3467 lines
165 KiB
Bash
Executable File
3467 lines
165 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ================================= COMMON LIBRARY =============================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Shared functions used by every script in the ecosystem.
|
||
# Sourced automatically by load_config.sh — do not source directly.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# A pure function library. It defines and returns; it starts nothing on its own.
|
||
#
|
||
# Load order matters and is owned by load_config.sh:
|
||
# varaverk.cfg → master.conf → host*.conf → common.sh → Plugin/$PLATFORM/adapter.sh
|
||
#
|
||
# common.sh is sourced AFTER the conf files because detect_hosts() and the health checks
|
||
# need HOST* and the thresholds to already exist. It is sourced BEFORE the adapter so the
|
||
# adapter can rely on log()/warn()/error() being defined.
|
||
#
|
||
# Note it defines detect_hosts() but never calls it. MY_ID and REMOTE_ID stay unset until a
|
||
# script calls it itself — anything building HOST*-prefixed variable names must do so after
|
||
# that call, not before.
|
||
#
|
||
# ==============================================================================================
|
||
# DESIGN PRINCIPLES
|
||
# ==============================================================================================
|
||
#
|
||
# Define, Never Act
|
||
# Sourcing this file changes no system state: no containers touched, no files written, no
|
||
# locks taken, no host detected. Every script in the ecosystem sources it, including
|
||
# read-only reporting ones, so anything that acted here would act everywhere.
|
||
#
|
||
# One Implementation of Each Shared Behaviour
|
||
# Locking, host detection, notification, retry and timeout wrapping live here once. When
|
||
# a safeguard needs strengthening it is strengthened for every caller at the same moment —
|
||
# which is why individual scripts call acquire_lock() rather than rolling their own flock.
|
||
#
|
||
# Platform-Agnostic
|
||
# No OS-specific paths or commands. Anything Unraid-specific belongs in the adapter, which
|
||
# is why this file calls platform_*() rather than touching /etc/rc.d or /boot directly.
|
||
#
|
||
# Callers Own Policy
|
||
# Helpers return status; they do not decide what failure means. retry_docker() reports
|
||
# that a command failed after N attempts — whether that aborts a run, skips an item, or
|
||
# raises a notification is the caller's call, and only the caller has the context.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# No Root, No Lock, No detect_hosts on Load — Deliberate
|
||
# This is a sourced library and must stay inert. A root check here would fire for every
|
||
# script including ones that legitimately do not need it; a lock would be taken on every
|
||
# source; calling detect_hosts() automatically would exit the caller on an unknown
|
||
# hostname before it had a chance to handle that itself. Do not add them.
|
||
#
|
||
# Safe Defaults on Every Threshold
|
||
# Helpers apply :- defaults (DOCKER_TIMEOUT, RESTART_VERIFY_WAIT, RETRY_COUNT and friends)
|
||
# so a conf missing a key degrades to a sane value rather than an empty string that would
|
||
# silently disable a timeout or a retry ceiling.
|
||
#
|
||
# Timeout Wrapping Provided Centrally
|
||
# docker_cmd() and the remote helpers wrap calls in timeout, so no caller has to remember
|
||
# to. A hung daemon or unreachable partner cannot stall a scheduled window.
|
||
#
|
||
# Locking Provided Centrally
|
||
# acquire_lock() implements strict, wait and continuous modes in one place, so every
|
||
# script gets identical semantics and a fix applies everywhere at once.
|
||
#
|
||
# Host Identity Is Exact-Match Only
|
||
# detect_hosts() matches hostname exactly, with a single explicit NetBIOS-truncation
|
||
# fallback that requires an unambiguous candidate. No fuzzy or similarity matching — a
|
||
# wrong host identity would alias the wrong credentials and paths into every script.
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# Defines no config of its own. It consumes what load_config.sh has already sourced from
|
||
# master.conf and host*.conf, and applies defaults where a key may be absent.
|
||
#
|
||
# Consumed broadly: DOCKER_TIMEOUT, RESTART_VERIFY_WAIT, RETRY_COUNT, SLEEP, SSH_KEY,
|
||
# SSH_TIMEOUT, STATE_DIR, DATA_DIR, LOCK_DIR, NOTIFY_UNRAID, HOST*, and the RSYNC_*/
|
||
# PARTNERSHIP_* gates.
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# None — sourced, never executed:
|
||
#
|
||
# source "$SCRIPT_DIR/../load_config.sh" # which sources this file
|
||
#
|
||
# It parses no arguments of its own. parse_args() is defined here for callers to invoke,
|
||
# and --dry-run/--status/--log are honoured by the calling script, not by this file.
|
||
#
|
||
# ==============================================================================================
|
||
# ── WHAT THIS FILE PROVIDES ───────────────────────────────────────────────────────────────────
|
||
# Icons — consistent visual language across all script output
|
||
# Output helpers — info, warn, error, success, log, notify
|
||
# Arg parser — unified --dry-run, --log, --status, KEY=VALUE handling
|
||
# Validation — require_var, validate_int
|
||
# Host detection — detect_hosts() identifies local/remote server, sets all aliases
|
||
# Connectivity — resolve_remote_ip, check_connectivity, ping_remote, ping_internet
|
||
# Health checks — local/remote array, docker daemon, rootfs, share, disk temps
|
||
# Disk verification — check_remote_disks() — XFS, ZFS, cache pools, nested paths
|
||
# Container mgmt — stop/start local and remote containers with state tracking
|
||
# Rsync helpers — get_rsync_opts, check_rsync_enabled
|
||
# Locking — acquire_lock, acquire_rsync_lock — concurrent execution protection
|
||
# Path translation — translate_path() — container → host path for arr cleanup scripts
|
||
# Arr utilities — check_arr_version() — API version safety gate
|
||
# Status display — show_status() — runtime config dump on --status flag
|
||
#
|
||
# ── CHANGELOG ─────────────────────────────────────────────────────────────────────────────────
|
||
#
|
||
# Version What changed
|
||
# ───────────────────────────────────────────────────────────────────────────────────────────
|
||
# v1.0 Initial stable framework — output helpers, arg parser, host detection
|
||
#
|
||
# v1.1 format_duration() moved here from daily_sync_maintenance.sh for shared use
|
||
# SSH_KEY naming collision resolved — gitea key renamed GITEA_SSH_KEY in master.conf
|
||
#
|
||
# v1.2 check_connectivity() added — fatal ping with Tailscale hint on failure
|
||
# Function header comment blocks standardised across all functions
|
||
#
|
||
# v1.3 check_remote_rootfs() — aborts rsync if remote rootfs exceeds ROOTFS_WARN
|
||
# check_remote_share() — aborts if target directory missing or empty on remote
|
||
# Both protect against rsync running when remote array is down
|
||
#
|
||
# v1.4 check_remote_disks() — verifies all physical disks backing a share are mounted
|
||
# Auto-discovers disk layout at runtime from disks.ini — no config required
|
||
# Aborts if any disk backing the share is offline or unmounted
|
||
#
|
||
# v1.5-2.9 Icon set expanded progressively — each operation has its own distinct icon
|
||
# All function output updated to use correct icon per context
|
||
# notify() added — unRAID native + Discord webhook notifications
|
||
# Script locking system added — acquire_lock(), acquire_rsync_lock()
|
||
# translate_path() added — container → host path for arr cleanup scripts
|
||
# check_arr_version() added — API version safety gate before arr operations
|
||
# check_api() added — pre-flight API reachability check
|
||
# ping_remote(), ping_internet() — non-fatal ping for fallback use
|
||
# check_remote_array() — array health check
|
||
# get_unraid_temp_thresholds() — reads thresholds from dynamix.cfg
|
||
# check_local_disk_temps() — pre-rsync temp check with exit codes 0/1/2
|
||
# stop/start local containers added alongside existing remote variants
|
||
#
|
||
# v3.1 Three-file config split — master.conf + host1.conf + host2.conf
|
||
# load_config.sh introduced — auto-discovers all host*.conf files
|
||
# detect_hosts() rewritten — sets MY_ID/REMOTE_ID and aliases all HOST* vars
|
||
# All scripts now source load_config.sh instead of conf files directly
|
||
# Adding a new server = add host*.conf, zero script changes required
|
||
#
|
||
# v3.2 check_remote_disks() rewritten — three-tier detection:
|
||
# Tier 1: array disk paths (/mnt/disk*/sharename)
|
||
# Tier 2: ZFS pools with find -maxdepth 2 (catches nested cache paths)
|
||
# Tier 3: shfs fallback — verifies /mnt/user is mounted
|
||
# Fixes: nested paths like /mnt/cache/appdata-Fallback/Critical-Data
|
||
# Fixes: ZFS cache pools not detected when share is not at pool root
|
||
#
|
||
# v3.3 PROFILE_REMOTE_RESTART_CONTAINERS support added
|
||
# Dirty sync profiles (critical-fallback) restart remote
|
||
# containers after sync if they were running before — picks up config changes
|
||
# Was stopped → stays stopped. Was running → gets restarted. ✅
|
||
#
|
||
# v3.5 notify_emby_scan() added — triggers Emby "Clean Missing Files" task
|
||
# after arr cleanup scripts delete orphaned files
|
||
# Emby immediately removes ghost entries — no user-facing file-not-found errors
|
||
# Called by lidarr/sonarr/radarr_cleanup.sh when files are deleted
|
||
#
|
||
#
|
||
# Two new safety functions added:
|
||
# check_os_version_parity() — refuses remote ops on version mismatch
|
||
# reads OS version via platform_get_os_version() / platform_os_version_probe_cmd()
|
||
# major mismatch → abort | minor mismatch → configurable warn/abort
|
||
# check_remote_docker_daemon() — verifies remote Docker daemon before
|
||
# issuing any remote container commands — strike system → skip/retry/exit
|
||
#
|
||
# ── VERSION ───────────────────────────────────────────────────────────────────────────────────
|
||
# Current: v3.5
|
||
# ==============================================================================================
|
||
|
||
# Cron runs with a minimal PATH (/usr/bin:/bin) that omits /usr/local/sbin where tailscale,
|
||
# and other Unraid tools live. Prepend it here so every sourcing script finds them.
|
||
export PATH="/usr/local/sbin:/usr/local/bin:$PATH"
|
||
|
||
# ==============================================================================================
|
||
# ── ICONS ─────────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Each icon has one job — do not reuse across different contexts.
|
||
# Adding a new icon: add to the appropriate group below with a comment describing its job.
|
||
|
||
# System / Host
|
||
ICON_HOST="🖥️" # host detection and identity
|
||
ICON_NET="🌐" # network / IP resolution
|
||
ICON_PING="📡" # connectivity check
|
||
ICON_GEAR="⚙️" # setup section header / profile load
|
||
|
||
# Health Checks
|
||
ICON_DISK="💾" # disk checks
|
||
ICON_HEALTH="🩺" # rootfs / share health checks
|
||
ICON_SHIELD="🛡️" # pre-flight section header
|
||
|
||
# Containers
|
||
ICON_CONTAINERS="📦" # container section anchor
|
||
ICON_STOP="⛔" # stop command being issued
|
||
ICON_STOPPED="🔴" # container confirmed stopped
|
||
ICON_START="▶️" # start command being issued
|
||
ICON_STARTED="💚" # container confirmed started
|
||
ICON_RUNNING="🟢" # container already running when checked
|
||
ICON_SKIP="⏭️" # skipping — already running healthy instance
|
||
ICON_PROTECTED="🔰" # file/item is protected — never delete
|
||
ICON_NOT_RUNNING="⭕" # container already stopped when checked
|
||
|
||
# Transfer
|
||
ICON_SYNC="🔄" # transfer section header
|
||
ICON_RUN="🚀" # sync starting / rsync attempt
|
||
ICON_RETRY="🔁" # retry attempt
|
||
ICON_DONE="🏁" # transfer complete
|
||
ICON_GIT="🌱" # git pull/push section header
|
||
|
||
# Summary
|
||
ICON_SUMMARY="📋" # summary section header
|
||
ICON_TIME="⏱️" # duration line
|
||
|
||
# System Operations
|
||
ICON_MOVER="🔃" # mover operations
|
||
ICON_REBOOT="⚡" # scheduled server reboot
|
||
ICON_REBOOT_SMART="🚨" # smart conditional reboot triggered
|
||
ICON_PLUGIN="🧩" # user scripts plugin operations
|
||
ICON_PHP="👥" # PHP-FPM operations
|
||
ICON_WEBGUI="💻" # WebGUI / nginx / emhttp operations
|
||
|
||
# Diagnostics
|
||
ICON_ZFS="📊" # ZFS ARC statistics
|
||
ICON_MEM="🧠" # memory status
|
||
ICON_WATCHDOG="🐾" # docker watchdog monitoring operations
|
||
|
||
# Media Operations
|
||
ICON_CLEAN="🧹" # media cleaner operations
|
||
ICON_TRASH="🗑️" # files being deleted
|
||
ICON_PERMS="🔐" # permissions operation / section header
|
||
ICON_UNLOCKED="🔓" # permissions successfully applied to a share
|
||
ICON_LOCK="🔏" # script instance lock — acquired/released
|
||
|
||
# Transcode Operations
|
||
ICON_RAM="💨" # ramdisk operations — fast ephemeral storage
|
||
ICON_LINK="🔗" # symlink state and management
|
||
|
||
# fallback Operations
|
||
ICON_FALLBACK="🔀" # fallback state changes and operations
|
||
|
||
# Docker Network Operations
|
||
ICON_DOCKER_NET="🔌" # Docker network connect operations
|
||
|
||
# Security / Certificate Operations
|
||
ICON_CERT="🔒" # SSL certificate monitoring
|
||
|
||
# Monitor Operations
|
||
ICON_MONITOR="📈" # monitoring section headers and general monitoring
|
||
ICON_SMART="🔧" # drive SMART health attribute monitoring
|
||
ICON_BANDWIDTH="📶" # bandwidth usage tracking and reporting
|
||
ICON_DIGEST="📰" # health digest — aggregated system summary
|
||
ICON_EMBY="🎬" # Emby media server session reporting
|
||
ICON_VERIFY="✔️" # backup verification — checksum comparison
|
||
|
||
# Notifications
|
||
ICON_NOTIFY="🔔" # notification operations
|
||
|
||
# Output
|
||
ICON_INFO="ℹ️"
|
||
ICON_WARN="⚠️"
|
||
ICON_ERROR="❌"
|
||
ICON_SUCCESS="✅"
|
||
|
||
# ==============================================================================================
|
||
# ── OUTPUT HELPERS ────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Standardised output functions used across all scripts.
|
||
#
|
||
# Output tiers:
|
||
# echo — always visible — summaries, status conclusions, section headers
|
||
# info() — always visible — status confirmations, reachability, check pass
|
||
# warn() — always visible — state transitions, warnings, important events
|
||
# error() — always visible — something broke
|
||
# success() — always visible — operation completed successfully
|
||
# log() — only with --log flag — per-item detail, internal checks
|
||
#
|
||
# All output goes to stdout — callers can redirect as needed.
|
||
|
||
info() { echo "$ICON_INFO [INFO] $*"; }
|
||
warn() { echo "$ICON_WARN [WARN] $*"; }
|
||
error() { echo "$ICON_ERROR [ERROR] $*"; }
|
||
success() { echo "$ICON_DONE [OK] $*"; }
|
||
|
||
log() {
|
||
[[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*"
|
||
return 0
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── NOTIFICATION ──────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Sends a notification via unRAID native system and/or Discord webhook.
|
||
# Both channels are optional and independently controlled:
|
||
# NOTIFY_UNRAID — shared toggle in master.conf
|
||
# MY_DISCORD_WEBHOOK — per-host in host*.conf, set by detect_hosts()
|
||
#
|
||
# Severity levels: normal, warning, alert
|
||
# Usage: notify "message" "subject" "severity"
|
||
|
||
notify() {
|
||
local message="$1"
|
||
local subject="${2:-unRAID Notification}"
|
||
local severity="${3:-normal}"
|
||
|
||
log "$ICON_NOTIFY Sending notification: $subject — $message"
|
||
|
||
# A dry run must not page anyone. It reaches this line by walking the same branches a real run
|
||
# would, so a --dry-run failover was one populated webhook away from alerting the operator at
|
||
# 4am about an outage nothing was acting on. The line above still records what WOULD have been
|
||
# sent, which is the part a preview is for.
|
||
if [[ "${DRY_RUN:-false}" == true ]]; then
|
||
log "$ICON_NOTIFY DRY RUN — notification not sent"
|
||
return 0
|
||
fi
|
||
|
||
if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then
|
||
if platform_send_os_notification "$message" "$subject" "$severity"; then
|
||
log "$ICON_NOTIFY OS notification sent"
|
||
else
|
||
log "$ICON_NOTIFY OS notify not available — skipping"
|
||
fi
|
||
fi
|
||
|
||
# Uses MY_DISCORD_WEBHOOK — set by detect_hosts() from HOST*_DISCORD_WEBHOOK
|
||
if [[ -n "${MY_DISCORD_WEBHOOK:-}" ]]; then
|
||
local payload
|
||
payload=$(printf '{"content": "%s — **%s**\\n%s"}' \
|
||
"$ICON_NOTIFY" "$subject" "$message")
|
||
if curl -s -H "Content-Type: application/json" \
|
||
-d "$payload" "$MY_DISCORD_WEBHOOK" >/dev/null 2>&1; then
|
||
log "$ICON_NOTIFY Discord notification sent"
|
||
else
|
||
warn "Discord notification failed — check HOST*_DISCORD_WEBHOOK in host*.conf"
|
||
fi
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── DURATION FORMATTER ────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Converts raw seconds into a human readable string.
|
||
# Examples: 47 → "47s" | 653 → "10m53s"
|
||
# Usage: format_duration $SECONDS
|
||
|
||
format_duration() {
|
||
local secs=$1
|
||
local days=$(( secs / 86400 ))
|
||
local hrs=$(( (secs % 86400) / 3600 ))
|
||
local mins=$(( (secs % 3600) / 60 ))
|
||
local rem=$(( secs % 60 ))
|
||
if [[ $days -gt 0 ]]; then echo "${days}d ${hrs}h ${mins}m"
|
||
elif [[ $hrs -gt 0 ]]; then echo "${hrs}h ${mins}m"
|
||
elif [[ $mins -gt 0 ]]; then echo "${mins}m${rem}s"
|
||
else echo "${rem}s"; fi
|
||
}
|
||
|
||
# Tiered human-readable size — picks B/MB/GB based on magnitude.
|
||
# Usage: format_bytes 5368709120 → 5.0GB
|
||
format_bytes() {
|
||
local bytes=${1:-0}
|
||
if (( bytes > 1073741824 )); then
|
||
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
|
||
elif (( bytes > 1048576 )); then
|
||
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
|
||
else
|
||
echo "${bytes}B"
|
||
fi
|
||
}
|
||
|
||
# Fixed-precision bytes → GB (no tiering) — for values always compared against a GB threshold.
|
||
# Usage: bytes_to_gb 5368709120 [decimals=2] → 5.00
|
||
bytes_to_gb() {
|
||
local bytes=${1:-0} decimals=${2:-2}
|
||
awk "BEGIN {printf \"%.${decimals}f\", $bytes / 1073741824}"
|
||
}
|
||
|
||
# Fixed-precision KB → GB (no tiering) — df --output=used/avail reports in KB.
|
||
# Usage: kb_to_gb 5242880 [decimals=2] → 5.00
|
||
kb_to_gb() {
|
||
local kb=${1:-0} decimals=${2:-2}
|
||
awk "BEGIN {printf \"%.${decimals}f\", $kb / 1048576}"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── ARG PARSER ────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Processes all flags and key=value pairs passed to any script.
|
||
# Call early in every script: parse_args "$@"
|
||
#
|
||
# Supported flags:
|
||
# --dry-run | -n → DRY_RUN=true
|
||
# --log → ENABLE_LOGGING=true
|
||
# --no-log → ENABLE_LOGGING=false
|
||
# --status → SHOW_STATUS=true
|
||
# --help | -h → print usage and exit
|
||
#
|
||
# Supported key=value:
|
||
# LOG=true/false → toggle logging
|
||
# BW_LIMIT=5000 → override any declared master.conf variable for this run
|
||
#
|
||
# Unparsed positional args returned in PARSED_ARGS array.
|
||
|
||
parse_args() {
|
||
ENABLE_LOGGING=${ENABLE_LOGGING:-false}
|
||
DRY_RUN=${DRY_RUN:-false}
|
||
SHOW_STATUS=${SHOW_STATUS:-false}
|
||
CLEAN_ARGS=()
|
||
|
||
for ARG in "$@"; do
|
||
if [[ "$ARG" == *=* ]]; then
|
||
VAR="${ARG%%=*}"
|
||
VAL="${ARG#*=}"
|
||
case "$VAR" in
|
||
LOG)
|
||
[[ "$VAL" == "true" ]] && ENABLE_LOGGING=true
|
||
[[ "$VAL" == "false" ]] && ENABLE_LOGGING=false
|
||
;;
|
||
*)
|
||
if declare -p "$VAR" &>/dev/null; then
|
||
printf -v "$VAR" '%s' "$VAL"
|
||
log "Set $VAR=$VAL"
|
||
else
|
||
warn "Unknown variable: $VAR"
|
||
fi
|
||
;;
|
||
esac
|
||
else
|
||
case "$ARG" in
|
||
--dry-run|-n) DRY_RUN=true ;;
|
||
--log) ENABLE_LOGGING=true ;;
|
||
--no-log) ENABLE_LOGGING=false ;;
|
||
--status|--summary) SHOW_STATUS=true ;;
|
||
--help|-h)
|
||
echo "Usage: script [--dry-run] [--log] [--status]"
|
||
exit 0
|
||
;;
|
||
*) CLEAN_ARGS+=("$ARG") ;;
|
||
esac
|
||
fi
|
||
done
|
||
|
||
PARSED_ARGS=("${CLEAN_ARGS[@]}")
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── VALIDATION HELPERS ────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Exits with error if a required variable is empty or unset.
|
||
# Usage: require_var VAR_NAME
|
||
require_var() {
|
||
[[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1
|
||
}
|
||
|
||
# Exits with error if a variable is not a valid positive integer.
|
||
# Usage: validate_int VAR_NAME "$VAR_VALUE"
|
||
validate_int() {
|
||
local name="$1" value="$2"
|
||
if [[ -z "$value" ]]; then
|
||
error "$name is not set — check master.conf"
|
||
exit 1
|
||
fi
|
||
if ! [[ "$value" =~ ^[0-9]+$ ]]; then
|
||
error "$name must be a positive integer — got: '$value'"
|
||
exit 1
|
||
fi
|
||
log "$name validated: $value"
|
||
}
|
||
|
||
# Membership check against a list of values (e.g. an ignore list).
|
||
# Usage: is_in_list "$needle" "${HAYSTACK_ARRAY[@]}"
|
||
is_in_list() {
|
||
local needle="$1"; shift
|
||
local item
|
||
for item in "$@"; do
|
||
[[ "$needle" == "$item" ]] && return 0
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── HOST DETECTION ────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Determines which server is local and which is remote by comparing hostname against
|
||
# all HOST* values discovered from host*.conf files.
|
||
#
|
||
# Sets:
|
||
# MY_ID — "HOST1" or "HOST2" (the role key, not the hostname)
|
||
# REMOTE_ID — the peer's role key
|
||
# LOCAL_SERVER_NAME — local hostname string (e.g. "unRAID-Gmer4Lfe")
|
||
# REMOTE_SERVER_NAME — remote hostname string
|
||
# SSH_KEY — SSH key for server-to-server operations
|
||
# REMOTE_STORAGE_PATH — remote's storage root from HOST*_STORAGE_PATH in host*.conf;
|
||
# falls back to platform_storage_path() for unconfigured pairs
|
||
#
|
||
# Also sets aliases for all host-specific arrays so scripts use unprefixed names:
|
||
# DAILY_SYNC_SHARES ← HOST*_DAILY_SYNC_SHARES
|
||
# PARTNERSHIP_PROVISION_SHARES ← HOST*_PARTNERSHIP_PROVISION_SHARES
|
||
# INTERMEDIATE_SYNC_SHARES ← HOST*_INTERMEDIATE_SYNC_SHARES
|
||
# WEEKLY_SYNC_SHARES ← HOST*_WEEKLY_SYNC_SHARES
|
||
# CRITICAL_SYNC_SHARES ← HOST*_CRITICAL_SYNC_SHARES
|
||
# DAILY_RESTART_CONTAINERS ← HOST*_DAILY_RESTART_CONTAINERS
|
||
# WEEKLY_RESTART_CONTAINERS ← HOST*_WEEKLY_RESTART_CONTAINERS
|
||
# WATCHDOG_CONTAINERS ← HOST*_WATCHDOG_CONTAINERS (associative)
|
||
# WATCHDOG_CONTAINER_URLS ← HOST*_WATCHDOG_CONTAINER_URLS (associative)
|
||
# WATCHDOG_REQUIRED_CONTAINERS ← HOST*_WATCHDOG_REQUIRED_CONTAINERS
|
||
# WATCHDOG_SCAN_IGNORE ← HOST*_WATCHDOG_SCAN_IGNORE
|
||
# WATCHDOG_DEPENDENCIES ← HOST*_WATCHDOG_DEPENDENCIES (associative)
|
||
# WATCHDOG_APPDATA_SIZES ← HOST*_WATCHDOG_APPDATA_SIZES (associative)
|
||
# NETWORK_CONNECT_CONTAINERS ← HOST*_NETWORK_CONNECT_CONTAINERS
|
||
# NETWORK_CONNECT_NETWORKS ← HOST*_NETWORK_CONNECT_NETWORKS
|
||
# MEDIA_PERMISSION_SHARES ← HOST*_MEDIA_PERMISSION_SHARES
|
||
# ANIME_CLEAN_FOLDERS ← HOST*_ANIME_CLEAN_FOLDERS
|
||
# MEDIA_CLEAN_FOLDERS ← HOST*_MEDIA_CLEAN_FOLDERS
|
||
# CERT_MONITOR_DOMAINS ← HOST*_CERT_MONITOR_DOMAINS
|
||
# SMART_IGNORE_DRIVES ← HOST*_SMART_IGNORE_DRIVES
|
||
# ZFS_REPORT_IGNORE_POOLS ← HOST*_ZFS_REPORT_IGNORE_POOLS
|
||
# TRANSCODE_SSD ← HOST*_TRANSCODE_SSD
|
||
# TRANSCODE_SERVERS ← HOST*_TRANSCODE_SERVERS
|
||
# RAMDISK_SIZE ← HOST*_RAMDISK_SIZE
|
||
# RAMDISK_WARN_GB ← HOST*_RAMDISK_WARN_GB
|
||
# RAMDISK_LOW_GB ← HOST*_RAMDISK_LOW_GB
|
||
# BACKUP_VERIFY_SHARES ← HOST*_BACKUP_VERIFY_SHARES
|
||
# DDNS_CONTAINERS ← HOST*_DDNS_CONTAINERS
|
||
# PARTNERSHIP_AUTH_WEBUIS ← HOST*_PARTNERSHIP_AUTH_WEBUIS
|
||
# PARTNERSHIP_MIRROR_BACKUPS ← HOST*_PARTNERSHIP_MIRROR_BACKUPS
|
||
# MY_DISCORD_WEBHOOK ← HOST*_DISCORD_WEBHOOK
|
||
# EMBY_CONTAINER ← HOST*_EMBY_CONTAINER
|
||
# EMBY_URL ← HOST*_EMBY_URL
|
||
# EMBY_API_KEY ← HOST*_EMBY_API_KEY
|
||
# JELLYFIN_CONTAINER ← HOST*_JELLYFIN_CONTAINER
|
||
# JELLYFIN_URL ← HOST*_JELLYFIN_URL
|
||
# JELLYFIN_API_KEY ← HOST*_JELLYFIN_API_KEY
|
||
# GITEA_API_TOKEN ← HOST*_GITEA_API_TOKEN
|
||
#
|
||
# Arr-specific vars set by detect_hosts() when needed by arr scripts:
|
||
# LIDARR_URL / LIDARR_API_KEY / LIDARR_MUSIC_ROOT (MY_ID only — Lidarr is HOST1 only)
|
||
# SONARR_URL / SONARR_API_KEY / SONARR_TV_ROOT
|
||
# RADARR_URL / RADARR_API_KEY / RADARR_MOVIES_ROOT
|
||
# SLSKD_URL / SLSKD_API_KEY / SLSKD_FAILED_IMPORTS_DIR (MY_ID only)
|
||
# SABNZBD_URL / SABNZBD_API_KEY (MY_ID only)
|
||
# QBIT_URL / QBIT_USERNAME / QBIT_PASSWORD (MY_ID only)
|
||
|
||
detect_hosts() {
|
||
local local_hostname
|
||
local_hostname="$(hostname)"
|
||
|
||
# ── Find MY_ID by matching hostname against all HOST* vars ────────────────
|
||
MY_ID=""
|
||
local host_var host_val
|
||
|
||
# Scan all HOST* vars that hold a hostname value
|
||
# HOST1, HOST2, HOST3 etc. — all defined in host*.conf files
|
||
for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||
host_val="${!host_var:-}"
|
||
[[ -z "$host_val" ]] && continue
|
||
if [[ "${local_hostname,,}" == "${host_val,,}" ]]; then
|
||
MY_ID="$host_var"
|
||
break
|
||
fi
|
||
done
|
||
|
||
# Unraid truncates the Server Name to 15 chars (NetBIOS limit). If the live hostname
|
||
# is at that exact limit, a configured HOST* value may be a longer, untruncated version
|
||
# (matching what Tailscale shows for this peer — resolve_tailscale_ip() keys off the same
|
||
# value). Only accept the match if it's unambiguous — exactly one HOST* may qualify.
|
||
if [[ -z "$MY_ID" && ${#local_hostname} -eq 15 ]]; then
|
||
local candidate="" candidate_count=0
|
||
for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||
host_val="${!host_var:-}"
|
||
[[ -z "$host_val" ]] && continue
|
||
if [[ ${#host_val} -gt 15 && "${host_val,,}" == "${local_hostname,,}"* ]]; then
|
||
candidate="$host_var"
|
||
(( candidate_count++ ))
|
||
fi
|
||
done
|
||
[[ "$candidate_count" -eq 1 ]] && MY_ID="$candidate"
|
||
fi
|
||
|
||
if [[ -z "$MY_ID" ]]; then
|
||
error "Unknown host: $local_hostname"
|
||
error "Hostname must match a HOST* value in host*.conf"
|
||
error "Available: $(for h in HOST1 HOST2 HOST3 HOST4; do
|
||
[[ -n "${!h:-}" ]] && echo -n "${!h} "; done)"
|
||
exit 1
|
||
fi
|
||
|
||
# ── Find REMOTE_ID — first HOST* that isn't MY_ID ────────────────────────
|
||
REMOTE_ID=""
|
||
for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||
[[ "$host_var" == "$MY_ID" ]] && continue
|
||
host_val="${!host_var:-}"
|
||
[[ -z "$host_val" ]] && continue
|
||
REMOTE_ID="$host_var"
|
||
break
|
||
done
|
||
|
||
# ── Set host name strings ─────────────────────────────────────────────────
|
||
LOCAL_SERVER_NAME="${!MY_ID}"
|
||
[[ -n "$REMOTE_ID" ]] && REMOTE_SERVER_NAME="${!REMOTE_ID}" || REMOTE_SERVER_NAME="unknown"
|
||
|
||
# ── Set SSH key ───────────────────────────────────────────────────────────
|
||
local ssh_key_var="${MY_ID}_SSH_KEY"
|
||
SSH_KEY="${!ssh_key_var:-}"
|
||
if [[ -z "$SSH_KEY" ]]; then
|
||
error "Missing SSH key: ${ssh_key_var} not set in host*.conf"
|
||
exit 1
|
||
fi
|
||
|
||
# ── Set scalar aliases ────────────────────────────────────────────────────
|
||
MY_DISCORD_WEBHOOK_VAR="${MY_ID}_DISCORD_WEBHOOK"
|
||
MY_DISCORD_WEBHOOK="${!MY_DISCORD_WEBHOOK_VAR:-}"
|
||
|
||
OWNER_NAME_VAR="${MY_ID}_OWNER"
|
||
OWNER_NAME="${!OWNER_NAME_VAR:-}"
|
||
OWNER_EMAIL_VAR="${MY_ID}_OWNER_EMAIL"
|
||
OWNER_EMAIL="${!OWNER_EMAIL_VAR:-}"
|
||
|
||
EMBY_CONTAINER_VAR="${MY_ID}_EMBY_CONTAINER"
|
||
EMBY_CONTAINER="${!EMBY_CONTAINER_VAR:-}"
|
||
EMBY_URL_VAR="${MY_ID}_EMBY_URL"
|
||
EMBY_URL="${!EMBY_URL_VAR:-}"
|
||
EMBY_API_KEY_VAR="${MY_ID}_EMBY_API_KEY"
|
||
EMBY_API_KEY="${!EMBY_API_KEY_VAR:-}"
|
||
|
||
JELLYFIN_CONTAINER_VAR="${MY_ID}_JELLYFIN_CONTAINER"
|
||
JELLYFIN_CONTAINER="${!JELLYFIN_CONTAINER_VAR:-}"
|
||
JELLYFIN_URL_VAR="${MY_ID}_JELLYFIN_URL"
|
||
JELLYFIN_URL="${!JELLYFIN_URL_VAR:-}"
|
||
JELLYFIN_API_KEY_VAR="${MY_ID}_JELLYFIN_API_KEY"
|
||
JELLYFIN_API_KEY="${!JELLYFIN_API_KEY_VAR:-}"
|
||
|
||
GITEA_API_TOKEN_VAR="${MY_ID}_GITEA_API_TOKEN"
|
||
GITEA_API_TOKEN="${!GITEA_API_TOKEN_VAR:-}"
|
||
|
||
local _prov_var
|
||
for _prov_var in \
|
||
PARTNERSHIP_PROVISION_EMBY_ADMIN \
|
||
PARTNERSHIP_EMBY_ADMIN_USER \
|
||
PARTNERSHIP_EMBY_ADMIN_PASS \
|
||
PARTNERSHIP_EMBY_PORT; do
|
||
local _src="${MY_ID}_${_prov_var}"
|
||
printf -v "$_prov_var" '%s' "${!_src:-}"
|
||
done
|
||
|
||
# ── System watchdog check toggles — per-host ─────────────────────────────
|
||
# Aliased as unprefixed SYS_WATCHDOG_CHECK_* for use in system_watchdog.sh
|
||
local _wd_checks=(
|
||
SYS_WATCHDOG_NIC
|
||
SYS_WATCHDOG_CHECK_DOCKER_DAEMON SYS_WATCHDOG_CHECK_ROOTFS
|
||
SYS_WATCHDOG_CHECK_KERNEL_OOPS SYS_WATCHDOG_CHECK_FD
|
||
SYS_WATCHDOG_CHECK_BOOT SYS_WATCHDOG_CHECK_OOM
|
||
SYS_WATCHDOG_CHECK_RAM SYS_WATCHDOG_CHECK_LOG
|
||
SYS_WATCHDOG_CHECK_ARC SYS_WATCHDOG_CHECK_CPU_TEMP
|
||
SYS_WATCHDOG_CHECK_LOAD SYS_WATCHDOG_CHECK_ZOMBIES
|
||
SYS_WATCHDOG_CHECK_CONTAINERS SYS_WATCHDOG_CHECK_TMP
|
||
SYS_WATCHDOG_CHECK_MDSTAT SYS_WATCHDOG_CHECK_NETWORK
|
||
SYS_WATCHDOG_CHECK_SSHD SYS_WATCHDOG_CHECK_RUNAWAY
|
||
)
|
||
local _wd_var
|
||
for _wd_var in "${_wd_checks[@]}"; do
|
||
local _wd_src="${MY_ID}_${_wd_var}"
|
||
# Only alias if the host-specific var is set — preserves master.conf defaults
|
||
[[ -n "${!_wd_src+x}" ]] && eval "${_wd_var}=\"\${${_wd_src}}\""
|
||
done
|
||
|
||
TRANSCODE_SSD_VAR="${MY_ID}_TRANSCODE_SSD"
|
||
TRANSCODE_SSD="${!TRANSCODE_SSD_VAR:-}"
|
||
RAMDISK_SIZE_VAR="${MY_ID}_RAMDISK_SIZE"
|
||
RAMDISK_SIZE="${!RAMDISK_SIZE_VAR:-8G}"
|
||
RAMDISK_WARN_GB_VAR="${MY_ID}_RAMDISK_WARN_GB"
|
||
RAMDISK_WARN_GB="${!RAMDISK_WARN_GB_VAR:-6.8}"
|
||
RAMDISK_LOW_GB_VAR="${MY_ID}_RAMDISK_LOW_GB"
|
||
RAMDISK_LOW_GB="${!RAMDISK_LOW_GB_VAR:-5.5}"
|
||
|
||
# Arr credentials — MY_ID only (remote arr accessed via its own host)
|
||
LIDARR_URL_VAR="${MY_ID}_LIDARR_URL"; LIDARR_URL="${!LIDARR_URL_VAR:-}"
|
||
LIDARR_API_KEY_VAR="${MY_ID}_LIDARR_API_KEY"; LIDARR_API_KEY="${!LIDARR_API_KEY_VAR:-}"
|
||
LIDARR_MUSIC_ROOT_VAR="${MY_ID}_LIDARR_MUSIC_ROOT"; LIDARR_MUSIC_ROOT="${!LIDARR_MUSIC_ROOT_VAR:-}"
|
||
LIDARR_RECOVERY_VAR="${MY_ID}_LIDARR_RECOVERY"; LIDARR_RECOVERY="${!LIDARR_RECOVERY_VAR:-false}"
|
||
FANART_API_KEY_VAR="${MY_ID}_FANART_API_KEY"; FANART_API_KEY="${!FANART_API_KEY_VAR:-}"
|
||
LASTFM_API_KEY_VAR="${MY_ID}_LASTFM_API_KEY"; LASTFM_API_KEY="${!LASTFM_API_KEY_VAR:-}"
|
||
TMDB_API_KEY_VAR="${MY_ID}_TMDB_API_KEY"; TMDB_API_KEY="${!TMDB_API_KEY_VAR:-}"
|
||
|
||
SONARR_URL_VAR="${MY_ID}_SONARR_URL"; SONARR_URL="${!SONARR_URL_VAR:-}"
|
||
SONARR_API_KEY_VAR="${MY_ID}_SONARR_API_KEY"; SONARR_API_KEY="${!SONARR_API_KEY_VAR:-}"
|
||
SONARR_TV_ROOT_VAR="${MY_ID}_SONARR_TV_ROOT"; SONARR_TV_ROOT="${!SONARR_TV_ROOT_VAR:-}"
|
||
SONARR_GENERAL_ROOT_VAR="${MY_ID}_SONARR_GENERAL_ROOT"; SONARR_GENERAL_ROOT="${!SONARR_GENERAL_ROOT_VAR:-}"
|
||
SONARR_KIDS_ROOT_VAR="${MY_ID}_SONARR_KIDS_ROOT"; SONARR_KIDS_ROOT="${!SONARR_KIDS_ROOT_VAR:-}"
|
||
SONARR_ANIME_ROOT_VAR="${MY_ID}_SONARR_ANIME_ROOT"; SONARR_ANIME_ROOT="${!SONARR_ANIME_ROOT_VAR:-}"
|
||
SONARR_RECOVERY_VAR="${MY_ID}_SONARR_RECOVERY"; SONARR_RECOVERY="${!SONARR_RECOVERY_VAR:-true}"
|
||
|
||
RADARR_URL_VAR="${MY_ID}_RADARR_URL"; RADARR_URL="${!RADARR_URL_VAR:-}"
|
||
RADARR_API_KEY_VAR="${MY_ID}_RADARR_API_KEY"; RADARR_API_KEY="${!RADARR_API_KEY_VAR:-}"
|
||
RADARR_MOVIES_ROOT_VAR="${MY_ID}_RADARR_MOVIES_ROOT"; RADARR_MOVIES_ROOT="${!RADARR_MOVIES_ROOT_VAR:-}"
|
||
RADARR_GENERAL_ROOT_VAR="${MY_ID}_RADARR_GENERAL_ROOT"; RADARR_GENERAL_ROOT="${!RADARR_GENERAL_ROOT_VAR:-}"
|
||
RADARR_KIDS_ROOT_VAR="${MY_ID}_RADARR_KIDS_ROOT"; RADARR_KIDS_ROOT="${!RADARR_KIDS_ROOT_VAR:-}"
|
||
RADARR_ANIME_ROOT_VAR="${MY_ID}_RADARR_ANIME_ROOT"; RADARR_ANIME_ROOT="${!RADARR_ANIME_ROOT_VAR:-}"
|
||
RADARR_RECOVERY_VAR="${MY_ID}_RADARR_RECOVERY"; RADARR_RECOVERY="${!RADARR_RECOVERY_VAR:-true}"
|
||
|
||
SLSKD_URL_VAR="${MY_ID}_SLSKD_URL"; SLSKD_URL="${!SLSKD_URL_VAR:-}"
|
||
SLSKD_API_KEY_VAR="${MY_ID}_SLSKD_API_KEY"; SLSKD_API_KEY="${!SLSKD_API_KEY_VAR:-}"
|
||
SLSKD_FAILED_IMPORTS_DIR_VAR="${MY_ID}_SLSKD_FAILED_IMPORTS_DIR"
|
||
SLSKD_FAILED_IMPORTS_DIR="${!SLSKD_FAILED_IMPORTS_DIR_VAR:-}"
|
||
|
||
SABNZBD_URL_VAR="${MY_ID}_SABNZBD_URL"; SABNZBD_URL="${!SABNZBD_URL_VAR:-}"
|
||
SABNZBD_API_KEY_VAR="${MY_ID}_SABNZBD_API_KEY"; SABNZBD_API_KEY="${!SABNZBD_API_KEY_VAR:-}"
|
||
|
||
QBIT_URL_VAR="${MY_ID}_QBIT_URL"; QBIT_URL="${!QBIT_URL_VAR:-}"
|
||
QBIT_USERNAME_VAR="${MY_ID}_QBIT_USERNAME"; QBIT_USERNAME="${!QBIT_USERNAME_VAR:-}"
|
||
QBIT_PASSWORD_VAR="${MY_ID}_QBIT_PASSWORD"; QBIT_PASSWORD="${!QBIT_PASSWORD_VAR:-}"
|
||
|
||
# ── Set array aliases — indexed arrays ────────────────────────────────────
|
||
# Each eval copies the host-specific array into the unprefixed name scripts use
|
||
|
||
_alias_array() {
|
||
local alias_name="$1"
|
||
local source_var="${MY_ID}_${alias_name}"
|
||
eval "${alias_name}=(\"\${${source_var}[@]}\")"
|
||
}
|
||
|
||
_alias_array "DAILY_SYNC_SHARES"
|
||
_alias_array "PERSONAL_SHARES"
|
||
_alias_array "INTERMEDIATE_SYNC_SHARES"
|
||
_alias_array "WEEKLY_SYNC_SHARES"
|
||
_alias_array "CRITICAL_SYNC_SHARES"
|
||
_alias_array "PARTNERSHIP_PROVISION_SHARES"
|
||
_alias_array "PARTNERSHIP_FALLBACK_ONLY"
|
||
_alias_array "BACKUP_VERIFY_SHARES"
|
||
_alias_array "DAILY_RESTART_CONTAINERS"
|
||
_alias_array "WEEKLY_RESTART_CONTAINERS"
|
||
_alias_array "WATCHDOG_REQUIRED_CONTAINERS"
|
||
_alias_array "WATCHDOG_SCAN_IGNORE"
|
||
_alias_array "NETWORK_CONNECT_CONTAINERS"
|
||
_alias_array "NETWORK_CONNECT_NETWORKS"
|
||
_alias_array "MEDIA_PERMISSION_SHARES"
|
||
_alias_array "ANIME_CLEAN_FOLDERS"
|
||
_alias_array "MEDIA_CLEAN_FOLDERS"
|
||
_alias_array "CERT_MONITOR_DOMAINS"
|
||
_alias_array "SMART_IGNORE_DRIVES"
|
||
_alias_array "ZFS_REPORT_IGNORE_POOLS"
|
||
_alias_array "TRANSCODE_SERVERS"
|
||
_alias_array "DDNS_CONTAINERS"
|
||
_alias_array "PARTNERSHIP_AUTH_WEBUIS"
|
||
_alias_array "PARTNERSHIP_MIRROR_BACKUPS"
|
||
_alias_array "PARTNERSHIP_OWN_CONTAINERS"
|
||
_alias_array "PARTNERSHIP_AUTH_STACK"
|
||
_alias_array "PARTNERSHIP_REPLACE_CONTAINERS"
|
||
_alias_array "PARTNERSHIP_ARR_STACK"
|
||
_alias_array "PARTNERSHIP_ARR_REPLACE_CONTAINERS"
|
||
_alias_array "PARTNERSHIP_SERVICES_STACK"
|
||
_alias_array "PARTNERSHIP_SERVICES_REPLACE_CONTAINERS"
|
||
_alias_array "RW_PAUSE_CONTAINERS"
|
||
_alias_array "RW_STOP_CONTAINERS"
|
||
_alias_array "PCIE_QUIET_DEVICES"
|
||
|
||
# ── Set array aliases — associative arrays ────────────────────────────────
|
||
# Associative arrays cannot be copied with eval — must be rebuilt key by key
|
||
|
||
_alias_assoc() {
|
||
local alias_name="$1"
|
||
local source_name="${MY_ID}_${alias_name}"
|
||
eval "declare -gA ${alias_name}"
|
||
local -n _assoc_src="${source_name}" 2>/dev/null || return 0
|
||
local -n _assoc_dst="${alias_name}"
|
||
local _assoc_key
|
||
for _assoc_key in "${!_assoc_src[@]}"; do
|
||
_assoc_dst["$_assoc_key"]="${_assoc_src[$_assoc_key]}"
|
||
done
|
||
}
|
||
|
||
_alias_assoc "WATCHDOG_CONTAINERS"
|
||
_alias_assoc "WATCHDOG_CONTAINER_URLS"
|
||
_alias_assoc "WATCHDOG_CONTAINER_API_CHECKS"
|
||
_alias_assoc "WATCHDOG_DEPENDENCIES"
|
||
_alias_assoc "WATCHDOG_APPDATA_SIZES"
|
||
|
||
# ── Remote storage path ───────────────────────────────────────────────────
|
||
# Resolved from REMOTE_ID's host*.conf so SSH commands use the correct path
|
||
# even when the remote is a different platform. Falls back to the local
|
||
# platform_storage_path() for unconfigured same-platform pairs.
|
||
local _rsp_var="${REMOTE_ID}_STORAGE_PATH"
|
||
REMOTE_STORAGE_PATH="${!_rsp_var:-$(platform_storage_path)}"
|
||
|
||
# ── Output ────────────────────────────────────────────────────────────────
|
||
log "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
|
||
}
|
||
|
||
# Derives owner/mirror roles from PARTNERSHIP_OWNER_HOST — used by every Partnership/*.sh
|
||
# script. Requires detect_hosts() to have already run (needs MY_ID, SSH_KEY).
|
||
# Sets: OWNER_ID MIRROR_ID OWNER MIRROR MIRROR_SSH_KEY OWNER_SSH_KEY AM_OWNER AM_MIRROR
|
||
# Does NOT set any *_STATE_FILE vars — callers that need those set them after calling this,
|
||
# since not every Partnership script needs the same set (transfer/onboard need fewer than
|
||
# manager/offboard).
|
||
partnership_resolve_roles() {
|
||
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
|
||
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
||
OWNER="${!OWNER_ID}"
|
||
MIRROR="${!MIRROR_ID}"
|
||
# SSH_KEY (set by detect_hosts) is this server's own private key. The remote accepts it
|
||
# because this server's PUBLIC key was installed there via ssh_setup.sh. With sparse
|
||
# checkout, each server only has its own host{N}.conf — the other server's key path is
|
||
# never available here. Use SSH_KEY for all outbound SSH regardless of mode.
|
||
MIRROR_SSH_KEY="$SSH_KEY"
|
||
OWNER_SSH_KEY="$SSH_KEY"
|
||
|
||
AM_OWNER=false
|
||
AM_MIRROR=false
|
||
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
||
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
||
}
|
||
|
||
# Read a scalar/array var from a partner's own config via SSH — the partner sources its
|
||
# own load_config.sh + detect_hosts() remotely so the value reflects ITS host*.conf, not
|
||
# ours. Requires $SCRIPT_DIR to be set by the caller (one level above the repo root, same
|
||
# convention every Partnership/*.sh script already uses for its own SCRIPT_DIR).
|
||
# Usage: read_remote_conf_var "$mirror_ip" "VAR_NAME"
|
||
# read_remote_conf_array "$mirror_ip" "ARRAY_NAME" # one element per line
|
||
read_remote_conf_var() {
|
||
local mirror_ip="$1" var_name="$2"
|
||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
|
||
detect_hosts 2>/dev/null
|
||
printf '%s' \"\${${var_name}:-}\"" 2>/dev/null
|
||
}
|
||
|
||
read_remote_conf_array() {
|
||
local mirror_ip="$1" var_name="$2"
|
||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
|
||
detect_hosts 2>/dev/null
|
||
printf '%s\n' \"\${${var_name}[@]:-}\"" 2>/dev/null
|
||
}
|
||
|
||
# Probes a remote host's SCRIPTS_DIR (in case it differs from ours — e.g. one host runs
|
||
# internal storage mode, the other flash), falling back to our own $SCRIPTS_DIR if the
|
||
# probe fails or the remote isn't reachable. strict_host_key defaults to "yes" (matches
|
||
# existing behavior everywhere except partnership_transfer.sh, which passes "no" since
|
||
# it may be contacting a mirror for the first time during an ownership transfer).
|
||
# Usage: resolve_remote_scripts_dir "$ip" ["$ssh_key"] ["no"|"yes"]
|
||
resolve_remote_scripts_dir() {
|
||
local ip="$1" ssh_key="${2:-$SSH_KEY}" strict_host_key="${3:-yes}"
|
||
local probe_cmd result
|
||
# SSH_TIMEOUT is set by the calling script, not by any conf, so a caller that forgets it
|
||
# hands `timeout` an empty interval — which fails instantly, before ssh is ever run. With
|
||
# stderr discarded the failure is invisible and the fallback below then returns THIS host's
|
||
# SCRIPTS_DIR as though it were the partner's. Every use of the result is a remote path, so
|
||
# the caller goes on to run commands against a directory that exists here and not there.
|
||
# That is how Step 1b ended up pointing at /boot on an appdata mirror.
|
||
local _to="${SSH_TIMEOUT:-15}"
|
||
local -a opts=(-o ConnectTimeout="$_to" -o BatchMode=yes)
|
||
[[ "$strict_host_key" == "no" ]] && opts+=(-o StrictHostKeyChecking=no)
|
||
probe_cmd=$(platform_scripts_dir_probe_cmd)
|
||
result=$(timeout "$_to" ssh -i "$ssh_key" "${opts[@]}" root@"$ip" "$probe_cmd" \
|
||
2>/dev/null | tr -d '[:space:]')
|
||
|
||
# A probe that answered is the only thing trusted. The fallback stays — callers need a path
|
||
# to name in an error message — but it is announced, because "we could not ask the partner
|
||
# where it keeps its scripts" and "the partner keeps them where we do" are different facts
|
||
# and were being reported identically.
|
||
if [[ -z "$result" ]]; then
|
||
warn "Could not read SCRIPTS_DIR from $ip — assuming $SCRIPTS_DIR, which is wrong if it runs from appdata" >&2
|
||
result="$SCRIPTS_DIR"
|
||
fi
|
||
echo "$result"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── MULTI-NODE DISCOVERY ──────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Scans HOST1..HOST8 for every configured node other than this one. Requires detect_hosts()
|
||
# to have already run (needs MY_ID). Sets the global REMOTE_NODES array — empty if none found.
|
||
discover_remote_nodes() {
|
||
REMOTE_NODES=()
|
||
local _hv
|
||
for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||
[[ "$_hv" == "$MY_ID" ]] && continue
|
||
[[ -z "${!_hv:-}" ]] && continue
|
||
REMOTE_NODES+=("$_hv")
|
||
done
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── REMOTE IP RESOLUTION ──────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Resolves the Tailscale IPv4 address of the remote server.
|
||
# Sets REMOTE_SERVER used by all subsequent SSH and rsync calls.
|
||
# Exits if resolution fails — Tailscale may be down or peer offline.
|
||
|
||
resolve_remote_ip() {
|
||
log "Resolving remote IP for $REMOTE_SERVER_NAME..."
|
||
local ts_name="${REMOTE_SERVER_NAME,,}"
|
||
local attempts=3
|
||
|
||
for ((i=1; i<=attempts; i++)); do
|
||
# Direct lookup — works when MagicDNS short-name resolution is active
|
||
REMOTE_SERVER=$(tailscale ip -4 "$ts_name" 2>/dev/null)
|
||
|
||
# Fallback — parse tailscale status for FQDN entries (e.g. hostname.tailXXXX.ts.net)
|
||
if [[ -z "$REMOTE_SERVER" ]]; then
|
||
REMOTE_SERVER=$(tailscale status 2>/dev/null | awk -v name="$ts_name" '$2 ~ "^" name { print $1; exit }')
|
||
fi
|
||
|
||
[[ -n "$REMOTE_SERVER" ]] && break
|
||
[[ $i -lt $attempts ]] && { warn "Tailscale resolution attempt $i/$attempts failed — retrying in 5s..."; sleep 5; }
|
||
done
|
||
|
||
if [[ -z "$REMOTE_SERVER" ]]; then
|
||
error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME after $attempts attempts"
|
||
error "Check: tailscale status | grep $ts_name"
|
||
exit 1
|
||
fi
|
||
info "$ICON_NET Remote IP: $REMOTE_SERVER"
|
||
}
|
||
|
||
# Resolve any hostname to a Tailscale IPv4 — status parse first, DNS as a bounded last resort.
|
||
# Usage: ip=$(resolve_tailscale_ip "hostname") — returns empty string on failure.
|
||
#
|
||
# `tailscale ip -4` used to be tried first, because Tailscale owns the mapping. It also resolves
|
||
# through MagicDNS, which does not work on this mesh — the two hosts are on separate tailnets
|
||
# shared into each other — so the name misses and the call blocks on a system DNS lookup until it
|
||
# times out. Five seconds, every call, on every rsync, partnership check and remote collector.
|
||
# The status parse costs ten milliseconds and carries the same mapping.
|
||
#
|
||
# Exactness is not traded away for the speed: an exact name match against `tailscale status` is as
|
||
# precise as the lookup it replaces. The prefix match keeps its ambiguity guard, and the DNS path
|
||
# still runs — bounded, and only when status cannot decide.
|
||
#
|
||
# Mirror of vv_resolve_tailscale_ip() in Plugin/unraid/include/config.php; the two must agree.
|
||
resolve_tailscale_ip() {
|
||
local hostname="${1,,}"
|
||
local status_out exact matches count ip
|
||
|
||
status_out=$(tailscale status 2>/dev/null)
|
||
|
||
exact=$(echo "$status_out" | awk -v name="$hostname" '
|
||
{ split(tolower($2), parts, "."); if (parts[1] == name) { print $1; exit } }')
|
||
if [[ -n "$exact" ]]; then
|
||
echo "$exact"
|
||
return
|
||
fi
|
||
|
||
# Unambiguous prefix match, either direction — handles Unraid's 15-char NetBIOS hostname
|
||
# truncation vs. a longer name recorded in master.conf. Only accept the match when exactly one
|
||
# peer could qualify; never guess between candidates that share a prefix (server1/server10).
|
||
matches=$(echo "$status_out" | awk -v name="$hostname" '
|
||
{ split(tolower($2), parts, "."); host = parts[1];
|
||
if (index(host, name) == 1 || index(name, host) == 1) print $1 }')
|
||
count=$(echo "$matches" | grep -c .)
|
||
if [[ "$count" -eq 1 ]]; then
|
||
echo "$matches"
|
||
return
|
||
fi
|
||
|
||
# Last resort, and the slow one. Bounded so a broken MagicDNS costs two seconds, not five.
|
||
ip=$(timeout 2 tailscale ip -4 "$hostname" 2>/dev/null)
|
||
[[ -n "$ip" ]] && echo "$ip"
|
||
}
|
||
|
||
# Strips the "unraid-" prefix (case-insensitive) and title-cases what remains.
|
||
# Usage: derive_short_name "unRAID-Gmer4Lfe" → "Gmer4lfe"
|
||
derive_short_name() {
|
||
local hostname="$1"
|
||
local short="${hostname,,}"
|
||
[[ "$short" == unraid-* ]] && short="${short:7}"
|
||
echo "${short^}"
|
||
}
|
||
|
||
# True if the given host*.conf basename belongs to this host (case-insensitive).
|
||
# Usage: is_own_conf_file "$(basename "$conf")"
|
||
is_own_conf_file() {
|
||
[[ "${1,,}" == "${MY_ID,,}.conf" ]]
|
||
}
|
||
|
||
# ── Set a boolean toggle in master.conf, in place ─────────────────────────────────────────────
|
||
# Booleans only. The value pattern stops at whitespace, so a quoted value containing spaces would
|
||
# be truncated — every gate this sets is true/false and nothing else should use it.
|
||
#
|
||
# Preserves what the line already looks like: leading indent, the existing quoting style, and the
|
||
# column the trailing comment sits in. master.conf is hand-aligned and read by a person; an edit
|
||
# that reflows a line makes a diff that looks bigger than the change actually is.
|
||
#
|
||
# Prefer this over update_master_conf() for toggles. That one rewrites the entire line as
|
||
# " KEY=VALUE", which drops the trailing comment — run it over the four sync gates and
|
||
# master.conf loses the notes explaining what each tier does.
|
||
#
|
||
# Lives here rather than in partnership_onboard.sh because onboard arming a gate and offboard
|
||
# disarming the same gate must be the same operation in both directions.
|
||
# Usage: set_conf_bool "$key" "true"|"false" "$master_conf_path"
|
||
set_conf_bool() {
|
||
local key="$1" val="$2" file="$3" cur pad spaces quoted gap
|
||
if ! grep -qE "^[[:space:]]*${key}=" "$file"; then
|
||
warn " ${key} is not in master.conf — skipped (add it to the template first)"
|
||
return 1
|
||
fi
|
||
cur=$(grep -E "^[[:space:]]*${key}=" "$file" | head -1 | sed -E "s|^[[:space:]]*${key}=||; s|[[:space:]]*#.*$||")
|
||
quoted=""; [[ "$cur" == \"*\" ]] && quoted='"'
|
||
cur="${cur//\"/}"
|
||
[[ "$cur" == "$val" ]] && { echo " ${key} already ${val}"; return 0; }
|
||
|
||
# Rewrite the whole gap rather than padding on top of the existing one. Adding
|
||
# (len(old) - len(new)) spaces in front of the gap that is already there holds the column for
|
||
# one edit and drifts on the next: false→true pads +1, true→false pads 0 but keeps that space,
|
||
# so every arm/disarm cycle pushes the comment one column right. An onboard and offboard pair
|
||
# is exactly that cycle, and master.conf is hand-aligned.
|
||
pad=0
|
||
if grep -qE "^[[:space:]]*${key}=[^#]*#" "$file"; then
|
||
# Existing run of spaces between the value and the #
|
||
gap=$(grep -E "^[[:space:]]*${key}=" "$file" | head -1 \
|
||
| sed -E "s|^[[:space:]]*${key}=[^#[:space:]]*||; s|#.*$||")
|
||
pad=$(( ${#gap} + ${#cur} - ${#val} ))
|
||
(( pad < 1 )) && pad=1
|
||
fi
|
||
spaces=$(printf '%*s' "$pad" '')
|
||
sed -i -E "s|^([[:space:]]*)${key}=[^#[:space:]]*[[:space:]]*(#.*)?$|\1${key}=${quoted}${val}${quoted}${spaces}\2|" "$file"
|
||
echo " ${key}: ${cur} → ${val}"
|
||
}
|
||
|
||
# Sets key=value in a flat state file (setup.db style) — updates in place if the
|
||
# key exists, appends if not. No indentation handling — for master.conf toggles use
|
||
# set_conf_bool() above.
|
||
# Usage: set_state_var "$state_file" "$key" "$value"
|
||
set_state_var() {
|
||
local file="$1" key="$2" value="$3"
|
||
if grep -q "^${key}=" "$file" 2>/dev/null; then
|
||
sed -i "s|^${key}=.*|${key}=${value}|" "$file"
|
||
else
|
||
echo "${key}=${value}" >> "$file"
|
||
fi
|
||
}
|
||
|
||
# Removes a key from a flat state file. The inverse of set_state_var, and the reason it exists:
|
||
# onboarding writes HOST*_PHASE*_DONE and nothing took them away again, so a completed offboard
|
||
# left a state file reading INACTIVE beside flags still claiming a finished onboard. Anything
|
||
# reading the flags — the setup checklist, the partnership card — believed the torn-down
|
||
# partnership was live.
|
||
#
|
||
# Absent key is success, not an error: clearing twice is how a re-run after a partial teardown
|
||
# should behave.
|
||
# Usage: clear_state_var "$state_file" "$key"
|
||
clear_state_var() {
|
||
local file="$1" key="$2"
|
||
[[ -f "$file" ]] || return 0
|
||
grep -q "^${key}=" "$file" 2>/dev/null || return 0
|
||
sed -i "/^${key}=/d" "$file"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── CONNECTIVITY CHECKS ───────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Fatal connectivity check — used by rsync and other scripts that must abort if unreachable.
|
||
# For fallback use ping_remote() which returns status without exiting.
|
||
check_connectivity() {
|
||
log "Checking connectivity to $REMOTE_SERVER..."
|
||
if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
|
||
error "$ICON_PING Remote $REMOTE_SERVER ($REMOTE_SERVER_NAME) is unreachable"
|
||
info "Hint: tailscale status | grep $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
info "$ICON_PING $REMOTE_SERVER_NAME is reachable"
|
||
}
|
||
|
||
# Non-fatal reachability checks — used by fallback.sh, fallback_test.sh, conf_cache_watchdog.sh.
|
||
# Return 0 if reachable, 1 if not. Never exit.
|
||
#
|
||
# Retried, because a single two-packet ping is too thin a basis for the decision it feeds.
|
||
# fallback.sh enters FALLBACK on ONE false from ping_remote — no strike count, by design, since
|
||
# every second of a real outage is downtime — so one dropped pair of ICMP packets performed a
|
||
# full failover: Tier 1 containers up, DDNS moved, operator notified, then a staged handback to
|
||
# undo it. Observed 2026-08-22 04:20 on a dry run: HOST2 had 3 weeks uptime and never missed a
|
||
# beat in its own loop, while this host declared it down and recovered 35 seconds later.
|
||
#
|
||
# Success returns immediately, so the healthy path — which is every cycle but the rare one —
|
||
# costs exactly what it did before. Only a failure pays for the retries.
|
||
#
|
||
# worst case = tries × (ping -c2 -W3 ≈ 4s) + (tries-1) × retry delay
|
||
# at the defaults: 3 × 4s + 2 × 2s = 16s, comfortably inside FALLBACK_CHECK_INTERVAL=30
|
||
#
|
||
# Raising retries past that budget would let one cycle overrun the next; the loop sleeps AFTER
|
||
# its work, so it would stretch the interval rather than overlap, but the detection latency is
|
||
# what the DDNS TTL is racing.
|
||
_ping_retry() {
|
||
local target="$1"
|
||
local tries="${FALLBACK_PROBE_RETRIES:-3}"
|
||
local gap="${FALLBACK_PROBE_RETRY_DELAY:-2}"
|
||
local i
|
||
for (( i = 1; i <= tries; i++ )); do
|
||
ping -c2 -W3 "$target" &>/dev/null && return 0
|
||
[[ "$i" -lt "$tries" ]] && sleep "$gap"
|
||
done
|
||
return 1
|
||
}
|
||
|
||
ping_remote() {
|
||
_ping_retry "$REMOTE_SERVER"
|
||
}
|
||
|
||
ping_internet() {
|
||
_ping_retry "${EXTERNAL_IP:-8.8.8.8}"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── RSYNC GATE CHECK ──────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Two-tier rsync enable/disable system.
|
||
# Tier 1: RSYNC_ENABLED — global gate, overrides everything
|
||
# Tier 2: orchestrator-specific flag — fine-grained control per window
|
||
#
|
||
# Usage:
|
||
# check_rsync_enabled "CRITICAL" ← checks RSYNC_ENABLED + CRITICAL_RSYNC_ENABLED
|
||
# check_rsync_enabled "INTERMEDIATE" ← checks RSYNC_ENABLED + INTERMEDIATE_RSYNC_ENABLED
|
||
# check_rsync_enabled "DAILY" ← checks RSYNC_ENABLED + DAILY_RSYNC_ENABLED
|
||
# check_rsync_enabled "WEEKLY" ← checks RSYNC_ENABLED + WEEKLY_RSYNC_ENABLED
|
||
# check_rsync_enabled "FALLBACK" ← checks RSYNC_ENABLED + FALLBACK_RSYNC_ENABLED
|
||
# check_rsync_enabled ← checks RSYNC_ENABLED only (direct rsync.sh call)
|
||
#
|
||
# Returns: 0 = enabled, proceed | 1 = disabled, skip cleanly
|
||
|
||
check_rsync_enabled() {
|
||
local orchestrator="${1:-}"
|
||
|
||
# Tier 1 — global gate
|
||
if [[ "${RSYNC_ENABLED:-true}" == false ]]; then
|
||
warn "RSYNC_ENABLED=false — rsync globally disabled, skipping all syncs"
|
||
return 1
|
||
fi
|
||
|
||
# Tier 2 — per-orchestrator
|
||
if [[ -n "$orchestrator" ]]; then
|
||
local var_name="${orchestrator}_RSYNC_ENABLED"
|
||
local var_value="${!var_name:-true}"
|
||
if [[ "$var_value" == false ]]; then
|
||
info "${var_name}=false — rsync disabled for $orchestrator orchestrator"
|
||
info "All other $orchestrator jobs will still run normally"
|
||
return 1
|
||
fi
|
||
fi
|
||
|
||
return 0
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── PLATFORM SERVICE STATE ────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Check whether Docker and VM Manager are enabled on this host.
|
||
# Delegates to platform_is_service_enabled() from the platform adapter.
|
||
|
||
is_docker_enabled() {
|
||
platform_is_service_enabled docker
|
||
}
|
||
|
||
is_vm_manager_enabled() {
|
||
platform_is_service_enabled libvirt
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── LOCAL HEALTH CHECKS ───────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# ==============================================================================================
|
||
# ── REMOTE HEALTH CHECKS ─────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Verifies remote storage is mounted via SSH.
|
||
# Non-fatal — returns status. Used before handback rsync.
|
||
# Syncing to remote with no array fills rootfs rapidly.
|
||
check_remote_array() {
|
||
log "Checking remote array on $REMOTE_SERVER_NAME..."
|
||
local _spath result
|
||
_spath="$REMOTE_STORAGE_PATH"
|
||
result=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||
"mountpoint -q '$_spath' && echo yes || echo no" 2>/dev/null)
|
||
if [[ "$result" != "yes" ]]; then
|
||
error "$ICON_DISK Remote array not started on $REMOTE_SERVER_NAME"
|
||
return 1
|
||
fi
|
||
log "Remote array is healthy"
|
||
return 0
|
||
}
|
||
|
||
# Aborts if remote rootfs (/) usage is at or above ROOTFS_WARN threshold.
|
||
# Fatal — exits the calling script.
|
||
# When remote array is down rsync writes land on rootfs and fill it rapidly.
|
||
check_remote_rootfs() {
|
||
log "Checking remote rootfs usage..."
|
||
local remote_usage
|
||
remote_usage=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null)
|
||
if [[ -z "$remote_usage" ]]; then
|
||
error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
if [[ "$remote_usage" -ge "${ROOTFS_WARN:-75}" ]]; then
|
||
error "$ICON_HEALTH Remote rootfs ${remote_usage}% — threshold ${ROOTFS_WARN:-75}%"
|
||
exit 1
|
||
fi
|
||
info "$ICON_HEALTH Remote rootfs: ${remote_usage}% (threshold: ${ROOTFS_WARN:-75}%)"
|
||
}
|
||
|
||
# Verifies target directory exists and is not empty on remote.
|
||
# Fatal — exits the calling script.
|
||
# Usage: check_remote_share "/mnt/user/Movies"
|
||
check_remote_share() {
|
||
local dir="$1"
|
||
log "Checking remote share: $dir..."
|
||
|
||
local share_exists
|
||
share_exists=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null)
|
||
if [[ "$share_exists" != "yes" ]]; then
|
||
error "$ICON_HEALTH Remote share does not exist: $dir"
|
||
exit 1
|
||
fi
|
||
|
||
local share_empty
|
||
share_empty=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null)
|
||
if [[ "$share_empty" == "yes" ]]; then
|
||
warn "$ICON_HEALTH Remote share exists but is empty: $dir — aborting to protect data"
|
||
exit 1
|
||
fi
|
||
|
||
info "$ICON_HEALTH Remote share verified: $dir"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── DISK TEMPERATURE CHECKS ───────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Returns 0 (true) if device is non-rotational (SSD/NVMe), 1 (false) if HDD.
|
||
# Checks /sys/block/<device>/queue/rotational — 0=SSD, 1=HDD.
|
||
is_ssd() {
|
||
local drive="$1"
|
||
local base rotational
|
||
base=$(basename "$drive")
|
||
rotational="/sys/block/$base/queue/rotational"
|
||
[[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0
|
||
# NVMe is always SSD — some NVMe controllers don't expose rotational correctly
|
||
[[ "$drive" == *nvme* ]] && return 0
|
||
return 1
|
||
}
|
||
|
||
# Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX
|
||
# Falls back to master.conf values (SMART_TEMP_WARN/CRIT) if platform returns defaults.
|
||
get_unraid_temp_thresholds() {
|
||
read -r UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX \
|
||
<<< "$(platform_get_temp_thresholds)"
|
||
UNRAID_DISK_HOT="${UNRAID_DISK_HOT:-${SMART_TEMP_WARN:-45}}"
|
||
UNRAID_DISK_MAX="${UNRAID_DISK_MAX:-${SMART_TEMP_CRIT:-55}}"
|
||
UNRAID_SSD_HOT="${UNRAID_SSD_HOT:-60}"
|
||
UNRAID_SSD_MAX="${UNRAID_SSD_MAX:-70}"
|
||
}
|
||
|
||
# Checks local disk temperatures before rsync.
|
||
# Fetches a domain's TLS cert expiry via openssl s_client + x509 and computes days
|
||
# remaining. Pure data — no logging, no thresholds — callers classify warn/crit
|
||
# themselves against their own CERT_WARN_DAYS/CERT_CRIT_DAYS and log however they like.
|
||
#
|
||
# Sets _CERT_DAYS, _CERT_EXPIRY (YYYY-MM-DD) on success. Sets _CERT_EXPIRY_RAW to the
|
||
# unparsed openssl date string only when parsing failed (empty when unreachable) so a
|
||
# caller that wants a more specific error message can distinguish the two failure modes.
|
||
#
|
||
# Usage: check_cert_expiry "$domain" [port=443] [timeout=$CERT_TIMEOUT or 10]
|
||
# Returns: 0 = fetched and parsed | 1 = unreachable or unparseable
|
||
check_cert_expiry() {
|
||
local domain="$1" port="${2:-443}" cert_timeout="${3:-${CERT_TIMEOUT:-10}}"
|
||
_CERT_DAYS=""
|
||
_CERT_EXPIRY=""
|
||
_CERT_EXPIRY_RAW=""
|
||
local expiry_str
|
||
expiry_str=$(echo | timeout "$cert_timeout" openssl s_client \
|
||
-connect "${domain}:${port}" -servername "$domain" \
|
||
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||
[[ -z "$expiry_str" ]] && return 1
|
||
local expiry_epoch
|
||
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
|
||
if [[ -z "$expiry_epoch" ]]; then
|
||
_CERT_EXPIRY_RAW="$expiry_str"
|
||
return 1
|
||
fi
|
||
_CERT_DAYS=$(( (expiry_epoch - $(date +%s)) / 86400 ))
|
||
_CERT_EXPIRY=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null)
|
||
return 0
|
||
}
|
||
|
||
# Reads temps and rotational flag from /var/local/emhttp/disks.ini.
|
||
# Uses unRAID's own thresholds from dynamix.cfg via get_unraid_temp_thresholds().
|
||
#
|
||
# Returns:
|
||
# 0 = all temps OK — proceed
|
||
# 1 = warn threshold exceeded — skip this profile only
|
||
# 2 = critical threshold exceeded — abort all remaining profiles
|
||
#
|
||
# Sets TEMP_CHECK_RESULT with human readable summary for logging.
|
||
check_local_disk_temps() {
|
||
get_unraid_temp_thresholds
|
||
|
||
local disks_ini_content
|
||
if ! disks_ini_content=$(platform_get_disk_states); then
|
||
warn "disk state unavailable — skipping temp check"
|
||
TEMP_CHECK_RESULT="temp check skipped (disk state unavailable)"
|
||
return 0
|
||
fi
|
||
|
||
local worst_result=0
|
||
local hot_drives=()
|
||
local crit_drives=()
|
||
local current_name=""
|
||
local current_device=""
|
||
local current_rotational=""
|
||
local current_temp=""
|
||
|
||
check_drive() {
|
||
[[ -z "$current_name" ]] || [[ -z "$current_temp" ]] && return
|
||
[[ "$current_temp" -eq 0 ]] && return # spun down — skip
|
||
|
||
local warn_thresh crit_thresh
|
||
if [[ "$current_rotational" == "0" ]]; then
|
||
warn_thresh="$UNRAID_SSD_HOT"
|
||
crit_thresh="$UNRAID_SSD_MAX"
|
||
else
|
||
warn_thresh="$UNRAID_DISK_HOT"
|
||
crit_thresh="$UNRAID_DISK_MAX"
|
||
fi
|
||
|
||
if [[ "$current_temp" -ge "$crit_thresh" ]]; then
|
||
crit_drives+=("${current_name}(${current_device}):${current_temp}°C≥${crit_thresh}°C")
|
||
[[ $worst_result -lt 2 ]] && worst_result=2
|
||
elif [[ "$current_temp" -ge "$warn_thresh" ]]; then
|
||
hot_drives+=("${current_name}(${current_device}):${current_temp}°C≥${warn_thresh}°C")
|
||
[[ $worst_result -lt 1 ]] && worst_result=1
|
||
fi
|
||
}
|
||
|
||
while IFS= read -r ini_line; do
|
||
if echo "$ini_line" | grep -qE '^\["(disk[0-9]+|parity[0-9]?|cache[0-9]?)"\]'; then
|
||
check_drive
|
||
current_name=$(echo "$ini_line" | grep -o '"[^"]*"' | head -1 | tr -d '"')
|
||
current_device=""
|
||
current_rotational="1" # default HDD
|
||
current_temp=""
|
||
elif echo "$ini_line" | grep -q '^device='; then
|
||
current_device=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
|
||
elif echo "$ini_line" | grep -q '^rotational='; then
|
||
current_rotational=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
|
||
elif echo "$ini_line" | grep -q '^temp='; then
|
||
current_temp=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
|
||
current_temp="${current_temp//[^0-9]/}"
|
||
current_temp="${current_temp:-0}"
|
||
fi
|
||
done <<< "$disks_ini_content"
|
||
check_drive # process last drive
|
||
|
||
if [[ ${#crit_drives[@]} -gt 0 ]]; then
|
||
TEMP_CHECK_RESULT="CRITICAL temps: ${crit_drives[*]}"
|
||
error "Drive temp CRITICAL — aborting all remaining syncs: ${crit_drives[*]}"
|
||
notify "Rsync aborted on $(hostname) — drive temp CRITICAL: ${crit_drives[*]}" "Rsync Temp Check" "warning"
|
||
return 2
|
||
elif [[ ${#hot_drives[@]} -gt 0 ]]; then
|
||
TEMP_CHECK_RESULT="HOT drives: ${hot_drives[*]}"
|
||
warn "Drive temp WARNING — skipping this profile: ${hot_drives[*]}"
|
||
notify "Rsync profile skipped on $(hostname) — drive temp WARNING: ${hot_drives[*]}" "Rsync Temp Check" "normal"
|
||
return 1
|
||
else
|
||
TEMP_CHECK_RESULT="all normal (HDD warn:${UNRAID_DISK_HOT}°C crit:${UNRAID_DISK_MAX}°C SSD warn:${UNRAID_SSD_HOT}°C crit:${UNRAID_SSD_MAX}°C)"
|
||
return 0
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── REMOTE DISK VERIFICATION ──────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Verifies all disks backing a share are healthy on the remote server.
|
||
# Auto-detects filesystem type from disks.ini — handles XFS, ZFS, and cache pools.
|
||
# No configuration required — discovers disk layout at runtime.
|
||
#
|
||
# Three-tier detection handles all share locations:
|
||
# Tier 1: Array disk paths — /mnt/disk*/sharename (XFS or ZFS per-disk)
|
||
# Tier 2: ZFS standalone pools — finds nested paths up to 2 levels deep
|
||
# catches /mnt/cache/appdata-Fallback/Critical-Data correctly
|
||
# Tier 3: shfs fallback — if share exists under /mnt/user, resolve backing pool
|
||
# or verify shfs itself is mounted if pool can't be determined
|
||
#
|
||
# Fatal — exits the calling script if any disk is offline.
|
||
# Usage: check_remote_disks "/mnt/user/Movies"
|
||
|
||
check_remote_disks() {
|
||
local dir="$1"
|
||
local share_name
|
||
share_name=$(basename "$dir")
|
||
|
||
info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..."
|
||
|
||
# Read remote disk state for fsType mapping
|
||
local disks_ini_content _disk_states_path
|
||
_disk_states_path=$(platform_disk_states_path)
|
||
disks_ini_content=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"cat '$_disk_states_path' 2>/dev/null" 2>/dev/null)
|
||
|
||
if [[ -z "$disks_ini_content" ]]; then
|
||
error "$ICON_DISK Cannot read disk state from $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
|
||
# Tier 1 — array disk paths (/mnt/disk*/sharename)
|
||
local backing_disks
|
||
backing_disks=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"ls -d /mnt/disk*/$share_name 2>/dev/null | awk -F/ '{print \$3}'" 2>/dev/null)
|
||
|
||
# Tier 2 — ZFS standalone pools, including nested paths
|
||
local zfs_pool_paths
|
||
zfs_pool_paths=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"zpool list -H -o name 2>/dev/null | while read pool; do
|
||
[[ -d \"/mnt/\${pool}/$share_name\" ]] && echo \"\$pool\" && continue
|
||
find \"/mnt/\${pool}\" -maxdepth 2 -name '$share_name' -type d 2>/dev/null | \
|
||
grep -q . && echo \"\$pool\"
|
||
done | sort -u" 2>/dev/null)
|
||
|
||
# Tier 3 — storage root fallback
|
||
local _spath="$REMOTE_STORAGE_PATH"
|
||
if [[ -z "$backing_disks" ]] && [[ -z "$zfs_pool_paths" ]]; then
|
||
local on_user
|
||
on_user=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"find '$_spath' -maxdepth 3 -name '$share_name' -type d 2>/dev/null | head -1" \
|
||
2>/dev/null)
|
||
if [[ -n "$on_user" ]]; then
|
||
local actual_path
|
||
actual_path=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"readlink -f '$on_user' 2>/dev/null || echo '$on_user'" 2>/dev/null)
|
||
local pool_name
|
||
pool_name=$(echo "$actual_path" | awk -F/ '{print $3}')
|
||
if [[ -n "$pool_name" ]] && [[ "$pool_name" != "user" ]]; then
|
||
zfs_pool_paths="$pool_name"
|
||
info "$ICON_DISK $share_name found at $actual_path on $REMOTE_SERVER_NAME"
|
||
else
|
||
# Can't determine pool — verify storage root itself is mounted
|
||
local shfs_ok
|
||
shfs_ok=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"mountpoint -q '$_spath' && echo yes || echo no" 2>/dev/null)
|
||
if [[ "$shfs_ok" == "yes" ]]; then
|
||
success "All disks backing $share_name are online ✅ (via storage root)"
|
||
return 0
|
||
fi
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
if [[ -z "$backing_disks" ]] && [[ -z "$zfs_pool_paths" ]]; then
|
||
error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
|
||
local all_ok=true
|
||
|
||
# Check array disks (XFS or ZFS single-disk-in-array)
|
||
if [[ -n "$backing_disks" ]]; then
|
||
while IFS= read -r disk_name; do
|
||
[[ -z "$disk_name" ]] && continue
|
||
|
||
local fs_type
|
||
fs_type=$(echo "$disks_ini_content" | awk -F= -v disk="$disk_name" '
|
||
/^\["'"'"'?/ { current=substr($0,3,length($0)-4) }
|
||
current==disk && /^fsType=/ { print $2; exit }
|
||
' | tr -d '"')
|
||
fs_type="${fs_type:-xfs}"
|
||
|
||
if [[ "$fs_type" == "zfs" ]]; then
|
||
local pool_health
|
||
pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"zpool list -H -o health '$disk_name' 2>/dev/null" 2>/dev/null)
|
||
if [[ "$pool_health" == "ONLINE" ]]; then
|
||
info "$ICON_DISK $disk_name (ZFS) $ICON_RUNNING — $share_name ONLINE"
|
||
else
|
||
error "$ICON_DISK $disk_name (ZFS) $ICON_STOPPED — pool ${pool_health:-offline}"
|
||
all_ok=false
|
||
fi
|
||
else
|
||
local mounted
|
||
mounted=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"mountpoint -q '/mnt/$disk_name' && echo yes || echo no" 2>/dev/null)
|
||
if [[ "$mounted" == "yes" ]]; then
|
||
info "$ICON_DISK $disk_name (XFS) $ICON_RUNNING — $share_name present"
|
||
else
|
||
error "$ICON_DISK $disk_name (XFS) $ICON_STOPPED — not mounted"
|
||
all_ok=false
|
||
fi
|
||
fi
|
||
done <<< "$backing_disks"
|
||
fi
|
||
|
||
# Check ZFS standalone pools (cache, gaming, media-servers etc.)
|
||
if [[ -n "$zfs_pool_paths" ]]; then
|
||
while IFS= read -r pool_name; do
|
||
[[ -z "$pool_name" ]] && continue
|
||
local pool_health
|
||
pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"zpool list -H -o health '$pool_name' 2>/dev/null" 2>/dev/null)
|
||
if [[ "$pool_health" == "ONLINE" ]]; then
|
||
info "$ICON_DISK $pool_name (ZFS pool) $ICON_RUNNING — $share_name ONLINE"
|
||
else
|
||
error "$ICON_DISK $pool_name (ZFS pool) $ICON_STOPPED — pool ${pool_health:-offline}"
|
||
all_ok=false
|
||
fi
|
||
done <<< "$zfs_pool_paths"
|
||
fi
|
||
|
||
if [[ "$all_ok" == false ]]; then
|
||
error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
success "All disks backing $share_name are online ✅"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── CONTAINER MANAGEMENT ──────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Four functions — stop/start on both local and remote.
|
||
# State tracking: only containers that WERE running get restarted.
|
||
# Was stopped → stays stopped. Was running → gets restarted. ✅
|
||
|
||
# Tracks which remote containers were running before stop — used by start_containers()
|
||
RUNNING_CONTAINERS=()
|
||
|
||
# ── Docker command helpers (used by docker_daily_restart, docker_weekly_restart, docker_update) ──
|
||
|
||
# Default timeouts — scripts can override before calling.
|
||
DOCKER_TIMEOUT=${DOCKER_TIMEOUT:-30} # seconds before docker command is killed
|
||
RESTART_VERIFY_WAIT=${RESTART_VERIFY_WAIT:-5} # seconds to wait before checking state post-restart
|
||
|
||
# Run a docker command with a hard timeout. Returns 124 on timeout, else docker's exit code.
|
||
docker_cmd() {
|
||
timeout "$DOCKER_TIMEOUT" "$@"
|
||
local exit_code=$?
|
||
if [[ "$exit_code" -eq 124 ]]; then
|
||
error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*"
|
||
return 1
|
||
fi
|
||
return "$exit_code"
|
||
}
|
||
|
||
# Verify a container is still running after a restart.
|
||
# Waits RESTART_VERIFY_WAIT seconds to let it settle, then checks State.Running.
|
||
verify_running() {
|
||
local container="$1"
|
||
sleep "$RESTART_VERIFY_WAIT"
|
||
local state
|
||
state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||
if [[ "$state" != "true" ]]; then
|
||
error "$container failed to stay running after restart — may have crashed"
|
||
return 1
|
||
fi
|
||
return 0
|
||
}
|
||
|
||
# Retry a docker command up to RETRY_COUNT times, sleeping SLEEP seconds between attempts.
|
||
retry_docker() {
|
||
local attempt=1
|
||
while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
|
||
log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
||
if docker_cmd "$@"; then
|
||
log "Succeeded on attempt $attempt"
|
||
return 0
|
||
else
|
||
warn "Attempt $attempt failed"
|
||
(( attempt++ ))
|
||
[[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP"
|
||
fi
|
||
done
|
||
error "Command failed after $RETRY_COUNT attempts: $*"
|
||
return 1
|
||
}
|
||
|
||
# ── Emby API helper ───────────────────────────────────────────────────────────
|
||
# General Emby REST call. Uses EMBY_URL and EMBY_API_KEY (aliased by detect_hosts).
|
||
# Usage: emby_api <endpoint> [timeout_seconds]
|
||
# Prints response body on 200; returns 1 on HTTP error or curl failure.
|
||
emby_api() {
|
||
local endpoint="$1" max_time="${2:-30}"
|
||
local response http_code body
|
||
response=$(curl -sf --max-time "$max_time" \
|
||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||
-w "\n%{http_code}" \
|
||
"${EMBY_URL}/${endpoint}" 2>/dev/null)
|
||
http_code=$(echo "$response" | tail -1)
|
||
body=$(echo "$response" | head -n -1)
|
||
[[ "$http_code" != "200" ]] && { error "Emby API HTTP $http_code: $endpoint"; return 1; }
|
||
echo "$body"
|
||
}
|
||
|
||
# ── Media Server Detection ────────────────────────────────────────────────────
|
||
# Picks whichever media server is actually running right now, so callers don't have to
|
||
# hardcode Emby or Jellyfin. Emby is preferred when both are up (it's the primary player on
|
||
# every host; Jellyfin is mainly kept around for its working ffprobe binary — see
|
||
# HOST*_FFPROBE_CONTAINER). Either container being merely *configured* in host*.conf doesn't
|
||
# count — this checks Docker's actual running state, so a host with Jellyfin installed but
|
||
# stopped correctly falls through to Emby (or to nothing, if neither is up).
|
||
#
|
||
# Requires detect_hosts() to have already run (needs EMBY_*/JELLYFIN_* aliases).
|
||
# Sets: MEDIA_SERVER ("Emby"|"Jellyfin"|""), MEDIA_SERVER_URL, MEDIA_SERVER_API_KEY.
|
||
# Returns 1 and leaves MEDIA_SERVER="" if neither configured container is running.
|
||
# Usage: detect_media_server [docker_timeout_seconds]
|
||
detect_media_server() {
|
||
local docker_timeout="${1:-10}"
|
||
MEDIA_SERVER="" MEDIA_SERVER_URL="" MEDIA_SERVER_API_KEY=""
|
||
|
||
local emby_up=false jellyfin_up=false
|
||
|
||
if [[ -n "${EMBY_CONTAINER:-}" ]]; then
|
||
[[ "$(timeout "$docker_timeout" docker inspect -f '{{.State.Running}}' \
|
||
"$EMBY_CONTAINER" 2>/dev/null)" == "true" ]] && emby_up=true
|
||
fi
|
||
if [[ -n "${JELLYFIN_CONTAINER:-}" ]]; then
|
||
[[ "$(timeout "$docker_timeout" docker inspect -f '{{.State.Running}}' \
|
||
"$JELLYFIN_CONTAINER" 2>/dev/null)" == "true" ]] && jellyfin_up=true
|
||
fi
|
||
|
||
if [[ "$emby_up" == true && "$jellyfin_up" == true ]]; then
|
||
log "Both Emby and Jellyfin are running — using Emby (primary)"
|
||
elif [[ "$emby_up" == true ]]; then
|
||
log "Emby is running, Jellyfin is not — using Emby"
|
||
elif [[ "$jellyfin_up" == true ]]; then
|
||
log "Jellyfin is running, Emby is not — using Jellyfin"
|
||
else
|
||
warn "Neither Emby nor Jellyfin is running — no media server available"
|
||
return 1
|
||
fi
|
||
|
||
if [[ "$emby_up" == true ]]; then
|
||
MEDIA_SERVER="Emby"
|
||
MEDIA_SERVER_URL="$EMBY_URL"
|
||
MEDIA_SERVER_API_KEY="$EMBY_API_KEY"
|
||
else
|
||
MEDIA_SERVER="Jellyfin"
|
||
MEDIA_SERVER_URL="$JELLYFIN_URL"
|
||
MEDIA_SERVER_API_KEY="$JELLYFIN_API_KEY"
|
||
fi
|
||
}
|
||
|
||
# General REST call against whichever server detect_media_server() picked. Jellyfin accepts
|
||
# the same X-Emby-Token header Emby uses (inherited from its Emby-fork lineage — confirmed
|
||
# live 2026-07-19), so one call shape covers both backends.
|
||
# Usage: detect_media_server && media_server_api <endpoint> [timeout_seconds]
|
||
# Prints response body on 200; returns 1 on HTTP error, curl failure, or no server detected.
|
||
media_server_api() {
|
||
local endpoint="$1" max_time="${2:-30}"
|
||
if [[ -z "${MEDIA_SERVER:-}" ]]; then
|
||
error "media_server_api called with no MEDIA_SERVER set — call detect_media_server first"
|
||
return 1
|
||
fi
|
||
local response http_code body
|
||
response=$(curl -sf --max-time "$max_time" \
|
||
-H "X-Emby-Token: $MEDIA_SERVER_API_KEY" \
|
||
-w "\n%{http_code}" \
|
||
"${MEDIA_SERVER_URL}/${endpoint}" 2>/dev/null)
|
||
http_code=$(echo "$response" | tail -1)
|
||
body=$(echo "$response" | head -n -1)
|
||
[[ "$http_code" != "200" ]] && { error "$MEDIA_SERVER API HTTP $http_code: $endpoint"; return 1; }
|
||
echo "$body"
|
||
}
|
||
|
||
# Stop containers on the REMOTE server via SSH.
|
||
# Reads CRITICAL_CONTAINER_NAMES — set from profile arrays by rsync.sh.
|
||
# Tracks running state in RUNNING_CONTAINERS for restart after sync.
|
||
stop_containers() {
|
||
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \
|
||
[[ "${CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then
|
||
log "No remote containers configured for this profile, skipping remote stop."
|
||
return
|
||
fi
|
||
echo "━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━"
|
||
info "Stopping remote containers..."
|
||
RUNNING_CONTAINERS=()
|
||
for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
local status
|
||
status=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "unknown")
|
||
if [[ "$status" == "true" ]]; then
|
||
echo "$ICON_STOP Stopping remote $c..."
|
||
RUNNING_CONTAINERS+=("$c")
|
||
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then
|
||
echo "$ICON_STOPPED $c stopped"
|
||
else
|
||
error "Failed to stop remote $c"
|
||
fi
|
||
elif [[ "$status" == "false" ]]; then
|
||
echo "$ICON_NOT_RUNNING $c is not running — skipping"
|
||
else
|
||
log "$c not found on remote — skipping"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Tracks which local containers were running before stop — used by start_local_containers()
|
||
LOCAL_RUNNING_CONTAINERS=()
|
||
|
||
# Stop containers on the LOCAL server.
|
||
# Reads LOCAL_CRITICAL_CONTAINER_NAMES — set from profile arrays by rsync.sh.
|
||
# Flushes local databases cleanly before pushing data to remote.
|
||
stop_local_containers() {
|
||
if [[ ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \
|
||
[[ "${LOCAL_CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then
|
||
log "No local containers configured for this profile, skipping local stop."
|
||
return
|
||
fi
|
||
info "Stopping local containers..."
|
||
LOCAL_RUNNING_CONTAINERS=()
|
||
for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
local status
|
||
status=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown")
|
||
if [[ "$status" == "true" ]]; then
|
||
echo "$ICON_STOP Stopping local $c..."
|
||
LOCAL_RUNNING_CONTAINERS+=("$c")
|
||
if docker stop "$c" >/dev/null; then
|
||
echo "$ICON_STOPPED $c stopped"
|
||
else
|
||
error "Failed to stop local $c"
|
||
fi
|
||
elif [[ "$status" == "false" ]]; then
|
||
echo "$ICON_NOT_RUNNING $c is not running — skipping"
|
||
else
|
||
log "$c not found locally — skipping"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Restart containers on the REMOTE server that were running before sync.
|
||
# Reads RUNNING_CONTAINERS set by stop_containers().
|
||
# Respects DELAYED_CONTAINERS — waits CONTAINER_DELAY seconds before starting them.
|
||
start_containers() {
|
||
if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then
|
||
log "No remote containers to restart."
|
||
return
|
||
fi
|
||
info "Starting remote containers..."
|
||
for c in "${RUNNING_CONTAINERS[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
local needs_delay=false
|
||
for d in "${DELAYED_CONTAINERS[@]}"; do
|
||
[[ "$c" == "$d" ]] && needs_delay=true && break
|
||
done
|
||
if [[ "$needs_delay" == true ]]; then
|
||
info "Waiting ${CONTAINER_DELAY}s before starting $c..."
|
||
sleep "$CONTAINER_DELAY"
|
||
fi
|
||
echo "$ICON_START Starting remote $c..."
|
||
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null 2>&1; then
|
||
echo "$ICON_STARTED $c started"
|
||
else
|
||
error "Failed to start remote $c — start manually if needed"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Restart containers on the LOCAL server that were running before sync.
|
||
# Reads LOCAL_RUNNING_CONTAINERS set by stop_local_containers().
|
||
# Respects DELAYED_CONTAINERS and CONTAINER_DELAY same as remote.
|
||
start_local_containers() {
|
||
if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then
|
||
log "No local containers to restart."
|
||
return
|
||
fi
|
||
info "Starting local containers..."
|
||
for c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
local needs_delay=false
|
||
for d in "${DELAYED_CONTAINERS[@]}"; do
|
||
[[ "$c" == "$d" ]] && needs_delay=true && break
|
||
done
|
||
if [[ "$needs_delay" == true ]]; then
|
||
info "Waiting ${CONTAINER_DELAY}s before starting local $c..."
|
||
sleep "$CONTAINER_DELAY"
|
||
fi
|
||
echo "$ICON_START Starting local $c..."
|
||
if docker start "$c" >/dev/null 2>&1; then
|
||
echo "$ICON_STARTED $c started"
|
||
else
|
||
error "Failed to start local $c — start manually if needed"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Builds a dependency-safe restart order — dependency containers (anything named as a
|
||
# value in WATCHDOG_DEPENDENCIES) restart first, then everything else in original order.
|
||
# Sets ORDERED_RESTART. Usage: build_restart_order CONTAINERS_ARRAY_NAME (bare name, e.g.
|
||
# build_restart_order DAILY_RESTART_CONTAINERS — not "${DAILY_RESTART_CONTAINERS[@]}").
|
||
build_restart_order() {
|
||
local -n _bro_containers="$1"
|
||
ORDERED_RESTART=()
|
||
local remaining=("${_bro_containers[@]}")
|
||
local placed=()
|
||
|
||
# First pass — add dependency containers that appear in our list
|
||
for container in "${remaining[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
local is_dependency=false
|
||
for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do
|
||
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
|
||
is_dependency=true
|
||
break
|
||
fi
|
||
done
|
||
if [[ "$is_dependency" == true ]]; then
|
||
local already=false
|
||
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
||
if [[ "$already" == false ]]; then
|
||
ORDERED_RESTART+=("$container")
|
||
placed+=("$container")
|
||
fi
|
||
fi
|
||
done
|
||
|
||
# Second pass — add remaining containers (dependents and independents)
|
||
for container in "${remaining[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
local already=false
|
||
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
||
if [[ "$already" == false ]]; then
|
||
ORDERED_RESTART+=("$container")
|
||
placed+=("$container")
|
||
fi
|
||
done
|
||
|
||
log "Restart order: ${ORDERED_RESTART[*]}"
|
||
}
|
||
|
||
# Waits CONTAINER_DELAY if this container depends on the last restarted one.
|
||
# Usage: check_dependency_delay "$container" "$last_restarted"
|
||
check_dependency_delay() {
|
||
local container="$1" last="$2"
|
||
[[ -z "$last" ]] && return
|
||
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
|
||
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
|
||
log "Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
|
||
sleep "$CONTAINER_DELAY"
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── RSYNC OPTIONS ─────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Loads rsync options for the current profile.
|
||
# Falls back to DEFAULT_RSYNC_OPTS from master.conf if no profile match found.
|
||
# Profile opts do NOT inherit from defaults — list all desired flags explicitly in master.conf.
|
||
|
||
get_rsync_opts() {
|
||
if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then
|
||
read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}"
|
||
log "Using profile rsync opts for $PROFILE_NAME: ${RSYNC_OPTS[*]}"
|
||
else
|
||
RSYNC_OPTS=("${DEFAULT_RSYNC_OPTS[@]}")
|
||
log "Using default rsync opts: ${RSYNC_OPTS[*]}"
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── SCRIPT LOCKING ────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Prevents multiple instances of the same script running simultaneously.
|
||
# All lock files live in /tmp/unraid_locks/ — auto-cleared on reboot.
|
||
#
|
||
# Modes:
|
||
# strict — exit immediately if already running (default)
|
||
# wait — wait LOCK_WAIT_TIMEOUT seconds then exit if still locked
|
||
# continuous — for long-running scripts: skip gracefully if healthy instance running
|
||
#
|
||
# Features:
|
||
# Stale lock detection — if lock PID is dead, clears and proceeds
|
||
# PID reuse protection — lock stores PID:scriptname, validates both
|
||
# Age warning — warns if lock older than LOCK_WARN_AGE (skipped for continuous)
|
||
# EXIT trap — lock always released on exit, crash, or kill signal
|
||
|
||
LOCK_DIR="/tmp/unraid_locks"
|
||
|
||
# Ensure DATA_DIR and STATE_DIR exist — created here so every script that sources common.sh
|
||
# can safely write to either without checking first.
|
||
if [[ -n "${DATA_DIR:-}" ]] && [[ ! -d "$DATA_DIR" ]]; then
|
||
mkdir -p "$DATA_DIR" 2>/dev/null || true
|
||
fi
|
||
if [[ -n "${STATE_DIR:-}" ]] && [[ ! -d "$STATE_DIR" ]]; then
|
||
mkdir -p "$STATE_DIR" 2>/dev/null || true
|
||
fi
|
||
|
||
# A cache, not a ledger. Nothing decrements it — acquire_rsync_lock() recounts live PIDs from the
|
||
# rsync_*.lock files and overwrites this with the truth before enforcing the limit, so the file
|
||
# routinely sits at a stale non-zero value with no rsync running. That is expected, not a leak:
|
||
# killed transfers cannot wedge the limit, because a lock whose PID is dead is not counted.
|
||
# Read it as "what the last acquire saw", never as "how many are running now".
|
||
RSYNC_COUNT_FILE="$LOCK_DIR/rsync_active_count"
|
||
RSYNC_MAX_CONCURRENT=3
|
||
LOCK_WARN_AGE=300 # seconds — warn if lock older than this (5min default)
|
||
LOCK_WAIT_TIMEOUT=30 # seconds — how long "wait" mode waits before giving up
|
||
|
||
# Registry of all lock files acquired by this process — released together on exit.
|
||
# Prevents the single-trap-per-acquire problem: each new acquire_lock/acquire_rsync_lock
|
||
# call used to overwrite the EXIT trap, orphaning the previous lock file.
|
||
declare -ga _LOCK_FILES=()
|
||
_release_all_locks() {
|
||
local lf
|
||
for lf in "${_LOCK_FILES[@]}"; do
|
||
[[ -f "$lf" ]] && rm -f "$lf"
|
||
done
|
||
}
|
||
_register_lock() {
|
||
_LOCK_FILES+=("$1")
|
||
trap '_release_all_locks' EXIT
|
||
}
|
||
|
||
# Internal — script name used as lock identifier
|
||
_lock_name() {
|
||
basename "${BASH_SOURCE[1]:-$0}" .sh
|
||
}
|
||
|
||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
# Timed mutes — a bounded, self-expiring exemption for one container
|
||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
# Every other exemption a watchdog offers is permanent: WATCHDOG_SCAN_IGNORE is a conf edit, an
|
||
# intentional stop lasts until it is cleared. So a container that needs quieting for an hour gets
|
||
# a workaround that outlives the reason for it, and nobody comes back — Healarr has sat in a
|
||
# pressure list since it was uninstalled, and seven names in the ignore list refer to containers
|
||
# that no longer exist. A mute is the missing shape: it states when it ends, and then it ends.
|
||
#
|
||
# Store format, one per line: container|until_epoch|reason
|
||
# Lives in STATE_DIR rather than conf because it is state, not configuration — and because a
|
||
# temporary decision written into conf is exactly the thing that stops being temporary.
|
||
#
|
||
# Fail-safe direction is deliberate and one-way: any problem reading the store yields NO mutes, so
|
||
# a corrupt or unreadable file makes the watchdogs behave normally rather than silently stop
|
||
# watching. Failing closed into silence is the one outcome a mute must never produce.
|
||
|
||
_wd_mute_file() {
|
||
echo "${WATCHDOG_MUTE_FILE:-${STATE_DIR:-/tmp}/watchdog_mutes.db}"
|
||
}
|
||
|
||
# Every container currently muted, one per line. Expired rows are simply not returned — expiry is
|
||
# a read-time comparison, so a mute ends on time whether or not anything prunes the file.
|
||
wd_mute_active() {
|
||
local f; f="$(_wd_mute_file)"
|
||
[[ -r "$f" ]] || return 0
|
||
local now; now=$(date +%s)
|
||
awk -F'|' -v now="$now" 'NF>=2 && $1!="" && $2+0>now {print $1}' "$f" 2>/dev/null
|
||
}
|
||
|
||
# Seconds remaining, or nothing when not muted. Used for reporting, never for control flow.
|
||
wd_mute_remaining() {
|
||
local f; f="$(_wd_mute_file)"
|
||
[[ -r "$f" ]] || return 0
|
||
local now; now=$(date +%s)
|
||
awk -F'|' -v now="$now" -v c="$1" 'NF>=2 && $1==c && $2+0>now {print $2-now; exit}' "$f" 2>/dev/null
|
||
}
|
||
|
||
wd_muted() {
|
||
[[ -n "$1" ]] || return 1
|
||
wd_mute_active | grep -qxF "$1"
|
||
}
|
||
|
||
# Drops expired rows. Called on every write so the file cannot grow without bound, and so the
|
||
# record on disk matches what is in force rather than accumulating history nobody reads.
|
||
wd_mute_prune() {
|
||
local f; f="$(_wd_mute_file)"
|
||
[[ -w "$f" ]] || return 0
|
||
local now tmp; now=$(date +%s); tmp="${f}.tmp.$$"
|
||
awk -F'|' -v now="$now" 'NF>=2 && $2+0>now' "$f" > "$tmp" 2>/dev/null && mv -f "$tmp" "$f"
|
||
rm -f "$tmp" 2>/dev/null
|
||
}
|
||
|
||
# wd_mute_add <container> <duration> [reason] duration: 45m | 2h | 1d
|
||
# Refuses anything past WATCHDOG_MUTE_MAX_HOURS. The cap is the whole point: without it this is
|
||
# just a slower way of writing an exemption that never expires.
|
||
wd_mute_add() {
|
||
local ctr="$1" dur="$2" reason="${3:-}"
|
||
[[ -n "$ctr" && -n "$dur" ]] || { echo "usage: wd_mute_add <container> <30m|2h|1d> [reason]" >&2; return 1; }
|
||
|
||
local n unit secs
|
||
n="${dur%[mhd]}"; unit="${dur##*[0-9]}"
|
||
[[ "$n" =~ ^[0-9]+$ && "$n" -gt 0 ]] || { echo "bad duration: $dur" >&2; return 1; }
|
||
case "$unit" in
|
||
m) secs=$(( n * 60 )) ;;
|
||
h) secs=$(( n * 3600 )) ;;
|
||
d) secs=$(( n * 86400 )) ;;
|
||
*) echo "bad duration unit: $dur (use m, h or d)" >&2; return 1 ;;
|
||
esac
|
||
|
||
local max=$(( ${WATCHDOG_MUTE_MAX_HOURS:-8} * 3600 ))
|
||
if (( secs > max )); then
|
||
echo "refused: ${dur} exceeds WATCHDOG_MUTE_MAX_HOURS=${WATCHDOG_MUTE_MAX_HOURS:-8}" >&2
|
||
return 1
|
||
fi
|
||
|
||
# A pipe would split the record and a newline would forge one, so neither is allowed through.
|
||
reason="${reason//|/ }"; reason="${reason//$'\n'/ }"
|
||
|
||
local f; f="$(_wd_mute_file)"
|
||
mkdir -p "$(dirname "$f")" 2>/dev/null
|
||
touch "$f" 2>/dev/null || { echo "cannot write $f" >&2; return 1; }
|
||
wd_mute_remove "$ctr" >/dev/null 2>&1 # re-muting replaces rather than stacks
|
||
printf '%s|%s|%s\n' "$ctr" "$(( $(date +%s) + secs ))" "$reason" >> "$f"
|
||
wd_mute_prune
|
||
}
|
||
|
||
wd_mute_remove() {
|
||
local ctr="$1" f; f="$(_wd_mute_file)"
|
||
[[ -n "$ctr" && -w "$f" ]] || return 0
|
||
local tmp="${f}.tmp.$$"
|
||
awk -F'|' -v c="$ctr" '$1!=c' "$f" > "$tmp" 2>/dev/null && mv -f "$tmp" "$f"
|
||
rm -f "$tmp" 2>/dev/null
|
||
}
|
||
|
||
# Internal — lock file path for this script
|
||
_lock_file() {
|
||
echo "$LOCK_DIR/${1:-$(_lock_name)}.lock"
|
||
}
|
||
|
||
# Acquire exclusive lock for this script.
|
||
# Usage: acquire_lock [strict|wait|continuous]
|
||
acquire_lock() {
|
||
local mode="${1:-strict}"
|
||
local script_name
|
||
script_name=$(basename "${BASH_SOURCE[1]:-$0}" .sh)
|
||
local lockfile
|
||
lockfile="$(_lock_file "$script_name")"
|
||
|
||
mkdir -p "$LOCK_DIR"
|
||
|
||
if [[ -f "$lockfile" ]]; then
|
||
local lock_content existing_pid locked_name
|
||
lock_content=$(cat "$lockfile" 2>/dev/null)
|
||
existing_pid="${lock_content%%:*}"
|
||
locked_name="${lock_content##*:}"
|
||
|
||
# Stale lock — PID dead
|
||
if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then
|
||
warn "Stale lock detected for $script_name (PID $existing_pid gone) — clearing"
|
||
rm -f "$lockfile"
|
||
|
||
# PID reuse — PID alive but belongs to different process
|
||
elif [[ "$locked_name" != "$script_name" ]]; then
|
||
warn "Lock PID $existing_pid reused by different process ($locked_name ≠ $script_name) — clearing"
|
||
rm -f "$lockfile"
|
||
|
||
else
|
||
# Lock is genuinely active — check age
|
||
local lock_age
|
||
lock_age=$(( $(date +%s) - $(stat -c %Y "$lockfile" 2>/dev/null || echo 0) ))
|
||
if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]] && [[ "$mode" != "continuous" ]]; then
|
||
warn "$script_name has been running for ${lock_age}s — may be stuck (PID $existing_pid)"
|
||
fi
|
||
|
||
if [[ "$mode" == "continuous" ]]; then
|
||
log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping"
|
||
exit 0
|
||
elif [[ "$mode" == "wait" ]]; then
|
||
info "Another instance of $script_name is running — waiting up to ${LOCK_WAIT_TIMEOUT}s"
|
||
local waited=0
|
||
while [[ -f "$lockfile" ]] && [[ "$waited" -lt "$LOCK_WAIT_TIMEOUT" ]]; do
|
||
sleep 1
|
||
((waited++))
|
||
lock_content=$(cat "$lockfile" 2>/dev/null)
|
||
existing_pid="${lock_content%%:*}"
|
||
locked_name="${lock_content##*:}"
|
||
if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then
|
||
warn "Lock became stale while waiting — clearing"
|
||
rm -f "$lockfile"
|
||
break
|
||
elif [[ "$locked_name" != "$script_name" ]]; then
|
||
warn "Lock PID reused while waiting — clearing"
|
||
rm -f "$lockfile"
|
||
break
|
||
fi
|
||
done
|
||
if [[ -f "$lockfile" ]]; then
|
||
error "$script_name still locked after ${LOCK_WAIT_TIMEOUT}s — exiting"
|
||
exit 1
|
||
fi
|
||
else
|
||
error "Another instance of $script_name is already running (PID $existing_pid) — exiting"
|
||
case "$script_name" in
|
||
fallback|transcode_management|daily_sync_maintenance|stability_watchdog)
|
||
notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning"
|
||
;;
|
||
esac
|
||
exit 1
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
# Acquire lock — store PID:scriptname to prevent PID reuse false positives
|
||
echo "$$:$script_name" > "$lockfile"
|
||
_register_lock "$lockfile"
|
||
log "$ICON_LOCK Lock acquired: $script_name (PID $$)"
|
||
}
|
||
|
||
# Acquire per-profile rsync lock + enforce global concurrent limit.
|
||
# Prevents same profile running twice and limits total concurrent rsync instances.
|
||
# Usage: acquire_rsync_lock "$PROFILE_NAME"
|
||
acquire_rsync_lock() {
|
||
local profile="$1"
|
||
local profile_lock
|
||
profile_lock="$(_lock_file "rsync_${profile}")"
|
||
|
||
mkdir -p "$LOCK_DIR"
|
||
|
||
# Per-profile lock — same profile cannot run twice
|
||
if [[ -f "$profile_lock" ]]; then
|
||
local lock_content existing_pid locked_name
|
||
lock_content=$(cat "$profile_lock" 2>/dev/null)
|
||
existing_pid="${lock_content%%:*}"
|
||
locked_name="${lock_content##*:}"
|
||
if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null && \
|
||
[[ "$locked_name" == "rsync_${profile}" ]]; then
|
||
error "rsync profile '$profile' is already running (PID $existing_pid) — exiting"
|
||
exit 1
|
||
else
|
||
warn "Stale rsync lock for profile '$profile' — clearing"
|
||
rm -f "$profile_lock"
|
||
fi
|
||
fi
|
||
|
||
# Global concurrent limit — validate count against actual live locks
|
||
local current_count=0
|
||
if [[ -f "$RSYNC_COUNT_FILE" ]]; then
|
||
current_count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 0)
|
||
local actual_count=0
|
||
for lf in "$LOCK_DIR"/rsync_*.lock; do
|
||
[[ -f "$lf" ]] || continue
|
||
local lpid lf_content
|
||
lf_content=$(cat "$lf" 2>/dev/null)
|
||
lpid="${lf_content%%:*}"
|
||
kill -0 "$lpid" 2>/dev/null && ((actual_count++))
|
||
done
|
||
if [[ "$actual_count" -ne "$current_count" ]]; then
|
||
log "rsync count corrected: $current_count → $actual_count"
|
||
current_count=$actual_count
|
||
echo "$current_count" > "$RSYNC_COUNT_FILE"
|
||
fi
|
||
fi
|
||
|
||
if [[ "$current_count" -ge "$RSYNC_MAX_CONCURRENT" ]]; then
|
||
error "Maximum concurrent rsync limit ($RSYNC_MAX_CONCURRENT) reached — exiting"
|
||
info "Active rsync locks: $(ls "$LOCK_DIR"/rsync_*.lock 2>/dev/null | \
|
||
xargs -I{} basename {} .lock | tr '\n' ' ')"
|
||
exit 1
|
||
fi
|
||
|
||
# Acquire profile lock and increment counter
|
||
echo "$$:rsync_${profile}" > "$profile_lock"
|
||
echo $(( current_count + 1 )) > "$RSYNC_COUNT_FILE"
|
||
_register_lock "$profile_lock"
|
||
log "$ICON_LOCK rsync lock acquired: profile '$profile' (PID $$, active: $(( current_count + 1 ))/$RSYNC_MAX_CONCURRENT)"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── FLAT STATE-FILE KEY/VALUE HELPERS ─────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Generic get/set for flat "key<sep>value" state files (one entry per line) — the pattern
|
||
# every watchdog's strike-tracking and state-file logic was independently reimplementing.
|
||
# grep -v + tempfile-swap on write, not sed -i in place — avoids sed treating a key containing
|
||
# regex metacharacters (container names, etc.) as part of the substitution pattern.
|
||
#
|
||
# Usage: wd_state_get "$key" "$file" [sep=:]
|
||
# wd_state_set "$key" "$value" "$file" [sep=:]
|
||
#
|
||
# Callers with a fixed state file and/or separator (e.g. stability_watchdog.sh's
|
||
# get_strikes/get_state_val, resource_watchdog.sh's rm_state_get/rm_state_get_eq) should
|
||
# keep their own thin same-named wrapper around these rather than changing call sites.
|
||
wd_state_get() {
|
||
local key="$1" file="$2" sep="${3:-:}"
|
||
grep -E "^${key}${sep}" "$file" 2>/dev/null | cut -d"$sep" -f2-
|
||
}
|
||
|
||
wd_state_set() {
|
||
local key="$1" value="$2" file="$3" sep="${4:-:}"
|
||
grep -vE "^${key}${sep}" "$file" 2>/dev/null > "${file}.tmp"
|
||
echo "${key}${sep}${value}" >> "${file}.tmp"
|
||
mv "${file}.tmp" "$file"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── ORCHESTRATOR CHILD EXECUTION ──────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Shared child-script runner for Orchestrators/*.sh. Replaces the run_job()/run_watchdog()
|
||
# copies that used to be hand-duplicated (with small inconsistencies) into each orchestrator —
|
||
# one implementation now, one place to fix a bug or extend behavior (e.g. per-child logging).
|
||
#
|
||
# Usage — caller declares its own tracking arrays before the loop:
|
||
# JOB_PASS=()
|
||
# JOB_FAIL=()
|
||
# for entry in "${SOME_SCRIPTS[@]}"; do
|
||
# run_orch_child "$entry"
|
||
# done
|
||
#
|
||
# $entry is "relative/path.sh [extra args...]" — the same format every master.conf
|
||
# script-list array already uses. Resolved against $ECOSYSTEM_ROOT, which the caller
|
||
# must set before calling this (the absolute repo root — "$(cd "$SCRIPT_DIR/.." && pwd)").
|
||
#
|
||
# --dry-run / --log are threaded down automatically from $DRY_RUN / $ENABLE_LOGGING —
|
||
# never $VERBOSE, which nothing in this codebase ever assigns.
|
||
|
||
# Writes the run record for a script an orchestrator ran, in the same shape and the same place
|
||
# run_job.sh writes one for a top-level job: LOG_DIR/<id>.json.
|
||
#
|
||
# Until this existed, only the ten jobs cron starts had a record — every script inside a
|
||
# *_MAINTENANCE_SCRIPTS, WATCHDOG_ORCHESTRATOR_SCRIPTS or ARRAY_*_SCRIPTS list had none, roughly
|
||
# seventy entries. Nothing announced that. Consumers just quietly got less: the arr cleanup stats
|
||
# fell through to a daily aggregate that cannot report an end time, and the troubleshoot profile
|
||
# answered "I cannot see a run record for that" for every script that is not an orchestrator —
|
||
# which is nearly all of them, and exactly the ones a question is usually about.
|
||
#
|
||
# Deliberately NOT run_job.sh. That wrapper also takes a lock, enforces a minimum interval and
|
||
# writes a manual sentinel; running it here would let an orchestrator's own child skip itself for
|
||
# being "too soon" or block on a lock the parent already holds. The record is the part that was
|
||
# missing, so the record is the only part this borrows.
|
||
#
|
||
# Failure is silent by design. A record that cannot be written must never stop the work it was
|
||
# describing — LOG_DIR could be read-only or full, and the script itself still needs to run.
|
||
_orch_child_record() {
|
||
local rel="${1%.sh}" start="$2" ec="${3-}"
|
||
[[ -n "$rel" && -n "${LOG_DIR:-}" ]] || return 0
|
||
|
||
local f="$LOG_DIR/$rel.json"
|
||
mkdir -p "$(dirname "$f")" 2>/dev/null || return 0
|
||
|
||
if [[ -z "$ec" ]]; then
|
||
printf '{"id":"%s.sh","status":"running","start":%s,"pid":%s}\n' \
|
||
"$rel" "$start" "$$" > "$f" 2>/dev/null
|
||
return 0
|
||
fi
|
||
|
||
# Same ladder run_job.sh uses: 0 ok, 1 warn, anything else error. Kept identical so a record
|
||
# written here and one written there mean the same thing to everything that reads them.
|
||
local status
|
||
if (( ec == 0 )); then status=ok
|
||
elif (( ec == 1 )); then status=warn
|
||
else status=error
|
||
fi
|
||
printf '{"id":"%s.sh","status":"%s","start":%s,"end":%s,"exit":%s}\n' \
|
||
"$rel" "$status" "$start" "$(date +%s)" "$ec" > "$f" 2>/dev/null
|
||
return 0
|
||
}
|
||
|
||
# Keeps a child's own log to the same ceiling run_job.sh keeps a job's: the last 1000 lines,
|
||
# replaced atomically. LOG_DIR is tmpfs, so an untrimmed chatty script spends RAM rather than disk,
|
||
# which is the worse of the two places to leak.
|
||
_orch_trim_log() {
|
||
local f="$1" n
|
||
[[ -f "$f" ]] || return 0
|
||
n=$(wc -l < "$f" 2>/dev/null) || return 0
|
||
(( n > 1000 )) || return 0
|
||
tail -n 1000 "$f" > "$f.tmp" 2>/dev/null && mv -f "$f.tmp" "$f" 2>/dev/null
|
||
rm -f "$f.tmp" 2>/dev/null
|
||
return 0
|
||
}
|
||
|
||
run_orch_child() {
|
||
local entry="$1"
|
||
local script_args script_path script_name label extra_args run_args
|
||
|
||
read -r -a script_args <<< "$entry"
|
||
script_path="$ECOSYSTEM_ROOT/${script_args[0]}"
|
||
script_name=$(basename "${script_args[0]}")
|
||
extra_args=("${script_args[@]:1}")
|
||
label="$script_name"
|
||
[[ -n "${extra_args[*]}" ]] && label="$script_name ${extra_args[*]}"
|
||
|
||
if [[ ! -f "$script_path" ]]; then
|
||
error "$script_name — not found at $script_path"
|
||
JOB_FAIL+=("$label")
|
||
return 1
|
||
fi
|
||
[[ ! -x "$script_path" ]] && chmod +x "$script_path"
|
||
|
||
run_args=("${extra_args[@]}")
|
||
[[ "$DRY_RUN" == true ]] && run_args+=("--dry-run")
|
||
[[ "$ENABLE_LOGGING" == true ]] && run_args+=("--log")
|
||
|
||
local _start _ec _clog
|
||
_start=$(date +%s)
|
||
log "Running: $label"
|
||
_orch_child_record "${script_args[0]}" "$_start"
|
||
|
||
# The child's output goes to its own log as well as the orchestrator's, so a script can be
|
||
# read on its own terms. Until now everything a child printed was interleaved into the
|
||
# parent's log with forty other scripts, and the only marker separating them was a human
|
||
# heading like "━━━ Media Cleaner — anime ━━━" that maps to no script id — which is why
|
||
# scoped_log could answer for ten orchestrators and nothing else.
|
||
#
|
||
# tee, not a redirect: the parent's log must keep receiving everything exactly as before, so
|
||
# this adds a destination rather than moving one. PIPESTATUS[0] is what preserves the child's
|
||
# exit code through the pipe — $? would report tee's.
|
||
#
|
||
# The whole thing is conditional on the log being writable, and falls back to the original
|
||
# unpiped call when it is not. A child must never fail, or be denied a run, because somewhere
|
||
# to write about it could not be opened.
|
||
_clog="${LOG_DIR:-}/${script_args[0]%.sh}.log"
|
||
if [[ -n "${LOG_DIR:-}" ]] && mkdir -p "$(dirname "$_clog")" 2>/dev/null \
|
||
&& : >> "$_clog" 2>/dev/null; then
|
||
printf '\n── %s ────────────────────────────────────────────────\n' \
|
||
"$(date '+%Y-%m-%d %H:%M:%S')" >> "$_clog" 2>/dev/null
|
||
bash "$script_path" "${run_args[@]}" 2>&1 | tee -a "$_clog"
|
||
_ec=${PIPESTATUS[0]}
|
||
_orch_trim_log "$_clog"
|
||
else
|
||
bash "$script_path" "${run_args[@]}"
|
||
_ec=$?
|
||
fi
|
||
|
||
if (( _ec == 0 )); then
|
||
log "$script_name — done in $(format_duration $(( $(date +%s) - _start )))"
|
||
_orch_child_record "${script_args[0]}" "$_start" 0
|
||
JOB_PASS+=("$label")
|
||
return 0
|
||
else
|
||
_orch_child_record "${script_args[0]}" "$_start" "$_ec"
|
||
# Match the severity ladder run_job.sh already records against: exit 1 is a warning,
|
||
# exit 2+ is an error. The status in the run record was always derived that way, so a
|
||
# child exiting 1 produced a run marked "warn" whose log was full of ❌ ERROR lines —
|
||
# and the log is the louder of the two. A job that skipped work it was told to skip read
|
||
# exactly like a job that broke.
|
||
#
|
||
# The label still lands in JOB_FAIL either way. Whether the child succeeded is a separate
|
||
# question from how loudly to report it, and the orchestrator's own exit code depends on
|
||
# the first one.
|
||
local _msg="$label — failed (exit $_ec, $(format_duration $(( $(date +%s) - _start ))))"
|
||
if (( _ec == 1 )); then warn "$_msg"; else error "$_msg"; fi
|
||
JOB_FAIL+=("$label")
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── PATH TRANSLATION ──────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Translates container-internal file paths to host paths for arr cleanup scripts.
|
||
# Arr APIs return file paths as seen inside the container — scripts need host paths to scan.
|
||
#
|
||
# Uses ARR_PATH_MAP associative array — must be declared before calling.
|
||
# Longest-match wins — prevents partial path collisions.
|
||
#
|
||
# Usage:
|
||
# declare -A ARR_PATH_MAP=(["/ext-music"]="/mnt/user/Music")
|
||
# translate_path "/ext-music/Artist/Album/track.flac"
|
||
# Returns: /mnt/user/Music/Artist/Album/track.flac
|
||
#
|
||
# If no match found — returns path unchanged (safe fallback).
|
||
|
||
translate_path() {
|
||
local api_path="$1"
|
||
local best_match=""
|
||
local best_len=0
|
||
|
||
for container_path in "${!ARR_PATH_MAP[@]}"; do
|
||
if [[ "$api_path" == "$container_path"* ]]; then
|
||
if [[ "${#container_path}" -gt "$best_len" ]]; then
|
||
best_match="$container_path"
|
||
best_len="${#container_path}"
|
||
fi
|
||
fi
|
||
done
|
||
|
||
if [[ -n "$best_match" ]]; then
|
||
echo "${ARR_PATH_MAP[$best_match]}${api_path#$best_match}"
|
||
else
|
||
echo "$api_path" # no match — return unchanged
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── CONTAINER STOP/RESTART FOR MAINTENANCE ────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# The "stop a container for a maintenance window, guarantee it comes back" shape shared by
|
||
# container_data_export.sh and emby_database_repair.sh. Callers still register their own EXIT
|
||
# trap (they may need extra cleanup, e.g. removing a partial archive) — the trap should call
|
||
# container_force_restart_if_needed() for the actual "still down? bring it back" part.
|
||
|
||
# Usage: container_stop_for_maintenance "$container" was_running_var_name \
|
||
# ["pre-stop reason message"] [log|warn] [post_stop_sleep=0]
|
||
# The reason message (if given) prints via the chosen level (default: log) right before the
|
||
# stop attempt, only when the container was actually running — callers use this for their
|
||
# own context ("stopping for clean export" vs "active sessions will be interrupted").
|
||
# Sets the named var true/false. Returns 1 (caller should exit) if the container doesn't
|
||
# exist or fails to stop; 0 otherwise (including the "wasn't running" case — caller logs
|
||
# its own context-specific message for that, since wording/visibility differs per caller).
|
||
container_stop_for_maintenance() {
|
||
local container="$1"
|
||
local -n _csm_was_running="$2"
|
||
local pre_stop_msg="${3:-}" pre_stop_level="${4:-log}" post_stop_sleep="${5:-0}"
|
||
_csm_was_running=false
|
||
|
||
local status
|
||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||
case "$status" in
|
||
true)
|
||
_csm_was_running=true
|
||
if [[ -n "$pre_stop_msg" ]]; then
|
||
if [[ "$pre_stop_level" == "warn" ]]; then warn "$pre_stop_msg"; else log "$pre_stop_msg"; fi
|
||
fi
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
if timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1; then
|
||
echo "$container stopped ✅"
|
||
[[ "$post_stop_sleep" -gt 0 ]] && sleep "$post_stop_sleep"
|
||
else
|
||
error "Failed to stop $container — aborting"
|
||
return 1
|
||
fi
|
||
else
|
||
warn "DRY RUN — would stop $container"
|
||
fi
|
||
;;
|
||
false) : ;; # not running — caller logs its own context-specific message
|
||
"")
|
||
error "$container not found — check container name"
|
||
return 1
|
||
;;
|
||
*)
|
||
warn "$container status: $status — proceeding with caution"
|
||
;;
|
||
esac
|
||
return 0
|
||
}
|
||
|
||
# Force-restart from inside a script's own EXIT trap if the container is still down after
|
||
# a crash mid-maintenance. No-op if it wasn't running before, in dry-run, or already up.
|
||
# Usage: container_force_restart_if_needed "$container" "$was_running"
|
||
container_force_restart_if_needed() {
|
||
local container="$1" was_running="$2"
|
||
[[ "$was_running" == true && "$DRY_RUN" == false ]] || return 0
|
||
local status
|
||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||
if [[ "$status" != "true" ]]; then
|
||
warn "Restarting $container (cleanup)..."
|
||
timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1 || \
|
||
error "Failed to restart $container — start it manually"
|
||
fi
|
||
}
|
||
|
||
# Normal-path restart after maintenance completes — verifies it stayed up, notifies on failure.
|
||
# Sets RESTART_OK true/false.
|
||
# Usage: container_restart_after_maintenance "$container" "$was_running" [settle_sleep=3] [notify_label]
|
||
container_restart_after_maintenance() {
|
||
local container="$1" was_running="$2" settle_sleep="${3:-3}" notify_label="${4:-Container Maintenance}"
|
||
RESTART_OK=false
|
||
|
||
if [[ "$was_running" != true ]]; then
|
||
echo "$container was not running — leaving stopped (state respected) ✅"
|
||
RESTART_OK=true
|
||
return 0
|
||
fi
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would restart $container"
|
||
RESTART_OK=true
|
||
return 0
|
||
fi
|
||
|
||
log "Restarting $container..."
|
||
if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then
|
||
sleep "$settle_sleep"
|
||
local post_status
|
||
post_status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||
if [[ "$post_status" == "true" ]]; then
|
||
echo "$container restarted and running ✅"
|
||
RESTART_OK=true
|
||
else
|
||
error "$container started but crashed — check container logs"
|
||
notify "$container failed to stay running after maintenance on $(hostname)" \
|
||
"$notify_label" "warning"
|
||
fi
|
||
else
|
||
error "Failed to restart $container — start it manually"
|
||
notify "$container failed to restart after maintenance on $(hostname)" \
|
||
"$notify_label" "warning"
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── ARR CLEANUP SAFETY GATES ──────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Shared by lidarr/radarr/sonarr_cleanup.sh — these scripts DELETE files, so every function
|
||
# here is a direct byte-for-byte port of what was independently duplicated three times, not
|
||
# a rewrite. Callers should behave identically to before this consolidation.
|
||
|
||
# Safety Layer 1 — container running, not starting/unhealthy. Exits 1 (with notify) on failure.
|
||
# Usage: check_container_health "$LIDARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Lidarr Cleanup"
|
||
check_container_health() {
|
||
local container="$1" docker_timeout="$2" notify_label="$3"
|
||
|
||
local running
|
||
running=$(timeout "$docker_timeout" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||
if [[ "$running" != "true" ]]; then
|
||
error "$container is not running — aborting"
|
||
notify "$notify_label aborted on $(hostname) — container not running" "$notify_label" "warning"
|
||
exit 1
|
||
fi
|
||
|
||
local health
|
||
health=$(timeout "$docker_timeout" docker inspect -f '{{.State.Health.Status}}' "$container" 2>/dev/null)
|
||
case "$health" in
|
||
healthy) info "$container is healthy" ;;
|
||
"") info "$container has no health check — proceeding" ;;
|
||
starting)
|
||
error "$container is still starting — aborting"
|
||
notify "$notify_label aborted on $(hostname) — container still starting" "$notify_label" "warning"
|
||
exit 1 ;;
|
||
unhealthy)
|
||
error "$container is unhealthy — aborting"
|
||
notify "$notify_label aborted on $(hostname) — container unhealthy" "$notify_label" "warning"
|
||
exit 1 ;;
|
||
*) warn "$container health: $health — proceeding with caution" ;;
|
||
esac
|
||
|
||
info "Safety layer 1 passed — container healthy"
|
||
}
|
||
|
||
# Safety Layer 6 — abort if tracked count dropped below min_pct of the last known count
|
||
# (protects against a misconfigured path / partial API response silently wiping the library).
|
||
# Writes the new baseline only if the check passes. Exits 1 (with notify) on failure.
|
||
# Usage: check_tracked_count_floor "$TRACKED_COUNT" "$LIDARR_TRACKED_COUNT_FILE" "$LIDARR_MIN_TRACKED_PCT" "Lidarr Cleanup"
|
||
check_tracked_count_floor() {
|
||
local tracked_count="$1" baseline_file="$2" min_pct="$3" notify_label="$4"
|
||
|
||
if [[ -f "$baseline_file" ]]; then
|
||
local last_count pct
|
||
last_count=$(cat "$baseline_file" 2>/dev/null || echo 0)
|
||
if [[ "$last_count" -gt 0 ]]; then
|
||
pct=$(awk "BEGIN {printf \"%d\", ($tracked_count / $last_count) * 100}")
|
||
if [[ "$pct" -lt "$min_pct" ]]; then
|
||
error "Tracked count dropped to ${pct}% of last run ($tracked_count vs $last_count)"
|
||
error "Suggests API issue — aborting to prevent mass deletion"
|
||
error "If expected (large library removal) delete: $baseline_file"
|
||
notify "$notify_label aborted on $(hostname) — tracked count dropped to ${pct}%" \
|
||
"$notify_label" "warning"
|
||
exit 1
|
||
fi
|
||
info "Tracked count: ${pct}% of last run ($tracked_count vs $last_count) ✅"
|
||
fi
|
||
else
|
||
info "No previous count on record — first run, saving baseline"
|
||
fi
|
||
|
||
echo "$tracked_count" > "$baseline_file"
|
||
}
|
||
|
||
# Filters --i-know-what-im-doing / --skip-age-check out of "$@" before parse_args sees them
|
||
# (both are cleanup-script-specific, not part of the shared arg parser).
|
||
# Sets I_KNOW, SKIP_AGE_CHECK, FILTERED_ARGS — call parse_args "${FILTERED_ARGS[@]}" after.
|
||
parse_destructive_flags() {
|
||
I_KNOW=false
|
||
SKIP_AGE_CHECK=false
|
||
FILTERED_ARGS=()
|
||
local arg
|
||
for arg in "$@"; do
|
||
case "$arg" in
|
||
--i-know-what-im-doing) I_KNOW=true ;;
|
||
--skip-age-check) SKIP_AGE_CHECK=true ;;
|
||
*) FILTERED_ARGS+=("$arg") ;;
|
||
esac
|
||
done
|
||
}
|
||
|
||
# Prints the nuclear-mode banner and sleeps 10s (Ctrl+C window) when both destructive flags
|
||
# are set and this isn't a dry run. Call after parse_args. No-op otherwise.
|
||
nuclear_mode_warning() {
|
||
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then
|
||
echo ""
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " Flags: --i-know-what-im-doing --skip-age-check"
|
||
echo " Age check: BYPASSED — deletes on first pass"
|
||
echo " Size threshold: BYPASSED — no GB limit"
|
||
echo " Data recovery: NOT POSSIBLE after deletion"
|
||
echo ""
|
||
echo " Review --dry-run output before proceeding."
|
||
echo " You have 10 seconds to cancel (Ctrl+C)..."
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
sleep 10
|
||
echo " Proceeding..."
|
||
echo ""
|
||
fi
|
||
}
|
||
|
||
# Safety Layer 7 — abort if total deletion size exceeds max_gb and --i-know-what-im-doing
|
||
# wasn't passed. Sets TOTAL_HUMAN. Exits 1 (with notify) on failure; warns and continues
|
||
# if I_KNOW is true (set by parse_destructive_flags).
|
||
# Usage: check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup"
|
||
check_delete_size_threshold() {
|
||
local total_bytes="$1" max_gb="$2" notify_label="$3"
|
||
local max_bytes
|
||
max_bytes=$(awk "BEGIN {printf \"%d\", $max_gb * 1073741824}")
|
||
if [[ "$total_bytes" -gt "$max_bytes" ]]; then
|
||
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $total_bytes / 1073741824}")
|
||
if [[ "$I_KNOW" != true ]]; then
|
||
echo ""
|
||
error "Deletion would exceed ${max_gb}GB — $TOTAL_HUMAN would be deleted"
|
||
error "Review ORPHAN lines above carefully before proceeding"
|
||
error "Rerun with: --i-know-what-im-doing"
|
||
error "To also bypass age check: add --skip-age-check"
|
||
notify "$notify_label halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
|
||
"$notify_label" "warning"
|
||
exit 1
|
||
else
|
||
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
|
||
fi
|
||
fi
|
||
}
|
||
|
||
# True if filepath's extension (case-insensitive) matches one of the given extensions.
|
||
# Usage: has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"
|
||
has_extension() {
|
||
local filepath="$1"; shift
|
||
local ext="${filepath##*.}"
|
||
ext="${ext,,}"
|
||
local valid_ext
|
||
for valid_ext in "$@"; do
|
||
[[ "$ext" == "$valid_ext" ]] && return 0
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# True if filepath's basename matches one of the given glob patterns.
|
||
# Usage: matches_pattern_list "$filepath" "${LIDARR_PROTECTED_PATTERNS[@]}"
|
||
#
|
||
# Parameter expansion instead of external basename (2026-07-17) — called once per
|
||
# non-tracked file in lidarr_cleanup.sh/sonarr_cleanup.sh/radarr_cleanup.sh's classification
|
||
# loop, which is most files in a media library (every protected sidecar: nfo/jpg/srt/etc).
|
||
# Measured: this was the actual dominant cost left in those scripts even after fixing the
|
||
# per-file stat fork and dirname elsewhere — 47.3s vs 1.65s for 10,000 calls (~28.6x),
|
||
# confirmed identical results across a spot check first.
|
||
matches_pattern_list() {
|
||
local filepath="$1"; shift
|
||
local filename="${filepath##*/}"
|
||
local pattern
|
||
for pattern in "$@"; do
|
||
# shellcheck disable=SC2254
|
||
case "$filename" in
|
||
$pattern) return 0 ;;
|
||
esac
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# Builds the global ARR_PATH_MAP associative array (container path → host path) from
|
||
# ${MY_ID}_<ARR_TYPE>_PATH_MAP, for translate_path() to use. Usage: build_arr_path_map "LIDARR"
|
||
build_arr_path_map() {
|
||
local arr_type="$1"
|
||
declare -gA ARR_PATH_MAP=()
|
||
local _bapm_var="${MY_ID}_${arr_type}_PATH_MAP"
|
||
eval "for key in \"\${!${_bapm_var}[@]}\"; do
|
||
ARR_PATH_MAP[\"\$key\"]=\"\${${_bapm_var}[\$key]}\"
|
||
done"
|
||
}
|
||
|
||
# Generic arr API GET wrapper with HTTP status check.
|
||
# Usage: arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "movie" "Radarr"
|
||
arr_api() {
|
||
local base_url="$1" api_key="$2" api_version="$3" endpoint="$4" label="${5:-Arr}"
|
||
local response http_code body
|
||
|
||
response=$(curl -sf \
|
||
--max-time 30 \
|
||
-H "X-Api-Key: $api_key" \
|
||
-w "\n%{http_code}" \
|
||
"${base_url}/api/${api_version}/${endpoint}" 2>/dev/null)
|
||
|
||
http_code=$(echo "$response" | tail -1)
|
||
body=$(echo "$response" | head -n -1)
|
||
|
||
if [[ "$http_code" != "200" ]]; then
|
||
error "$label API HTTP $http_code for: $endpoint"
|
||
return 1
|
||
fi
|
||
echo "$body"
|
||
}
|
||
|
||
# Triggers an arr "command" endpoint with the given JSON payload, then polls every 10s until
|
||
# completed/failed/timeout, logging progress every 60s. Best-effort pre-flight — never treats
|
||
# an unreachable/timed-out scan as fatal, matching original per-script behavior.
|
||
#
|
||
# On a genuine "completed" observation, records the actual elapsed duration keyed by the
|
||
# command's own name via arr_record_rescan_duration() (needs arr_type to know which arr's
|
||
# duration DB to write to) — this is the one place in the codebase that reliably watches a
|
||
# command from trigger to completion, so it's the natural spot to build up real historical
|
||
# duration data for the wait-calibration logic in arr_get_tracked_data(). A timeout doesn't
|
||
# record anything — we only know a lower bound, not the true duration, and recording that
|
||
# would corrupt future wait calculations downward. arr_type is optional — omit it (or pass
|
||
# empty) to skip duration recording entirely, e.g. for one-off commands that aren't one of
|
||
# the three tracked arrs.
|
||
#
|
||
# Usage: trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" \
|
||
# '{"name": "DownloadedAlbumsScan"}' "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}" "lidarr"
|
||
trigger_and_await_command() {
|
||
local base_url="$1" api_key="$2" api_version="$3" payload="$4" poll_timeout="${5:-600}" arr_type="${6:-}"
|
||
|
||
local cmd_name
|
||
cmd_name=$(echo "$payload" | jq -r '.name // empty' 2>/dev/null)
|
||
|
||
local scan_response scan_cmd_id
|
||
scan_response=$(curl -sf --max-time 30 -X POST \
|
||
-H "X-Api-Key: $api_key" \
|
||
-H "Content-Type: application/json" \
|
||
-d "$payload" \
|
||
"${base_url}/api/${api_version}/command" 2>/dev/null)
|
||
|
||
scan_cmd_id=$(echo "$scan_response" | jq -r '.id // empty' 2>/dev/null)
|
||
|
||
if [[ -z "$scan_cmd_id" ]]; then
|
||
warn "Could not trigger import scan — proceeding without pre-flight"
|
||
return 0
|
||
fi
|
||
|
||
info "Import scan queued (command ID: $scan_cmd_id) — waiting for completion..."
|
||
local start_epoch=$(date +%s)
|
||
local polled=0 scan_status
|
||
while [[ "$polled" -lt "$poll_timeout" ]]; do
|
||
scan_status=$(curl -sf --max-time 10 \
|
||
-H "X-Api-Key: $api_key" \
|
||
"${base_url}/api/${api_version}/command/${scan_cmd_id}" 2>/dev/null | \
|
||
jq -r '.status // empty' 2>/dev/null)
|
||
case "$scan_status" in
|
||
completed)
|
||
info "Import scan complete ✅"
|
||
[[ -n "$cmd_name" && -n "$arr_type" ]] && \
|
||
arr_record_rescan_duration "$arr_type" "$cmd_name" "$(( $(date +%s) - start_epoch ))"
|
||
return 0
|
||
;;
|
||
failed)
|
||
warn "Import scan reported failed — proceeding anyway"
|
||
return 0
|
||
;;
|
||
esac
|
||
sleep 10
|
||
(( polled += 10 ))
|
||
[[ $(( polled % 60 )) -eq 0 ]] && log " Still scanning... (${polled}s elapsed)"
|
||
done
|
||
warn "Import scan timed out after ${poll_timeout}s — proceeding anyway"
|
||
return 0
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── ARR TRACKED-DATA CACHE (Lidarr, Sonarr, Radarr) ──────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Shared cache for each arr's tracked-library data (Lidarr artists, Sonarr series, Radarr
|
||
# movies). Every script across all three arrs that needs "what does this arr think it has
|
||
# tracked" goes through this — 15+ consumers as of 2026-07-17 (all three cleanup scripts, the
|
||
# missing-art/release-fixer/duplicate-cleanup scripts, all three playback-aware discovery
|
||
# scripts, the TVDB/TMDb-removed checks, arr_sync.sh's local side, and the emby_to_*_sync
|
||
# bootstrap tools). Hitting the live API fresh every time is wasteful, and during an active
|
||
# rescan the live number is actively misleading — tracked counts dip and recover as files are
|
||
# detached/re-verified one by one (confirmed 2026-07-16 on Lidarr: a whole-library
|
||
# RescanFolders made trackFileCount read 22% of normal mid-scan, which is exactly the
|
||
# false-alarm lidarr_cleanup.sh's count-drop guard is meant to catch, but a genuine rescan
|
||
# isn't the "something's actually wrong" case that guard exists for — the same risk applies to
|
||
# Sonarr's RescanSeries and Radarr's RescanMovie).
|
||
#
|
||
# Built Lidarr-only first (2026-07-16), generalized the same day to cover all three arrs —
|
||
# identical mechanism, keyed by arr_type ("lidarr"/"sonarr"/"radarr") so each arr's cache and
|
||
# duration history stay separate.
|
||
#
|
||
# Storage (2026-07-17): primary copy lives on tmpfs (ARR_CACHE_DIR) — reads/writes never touch
|
||
# the array disk, and losing it on reboot costs nothing since a full rebuild for all three arrs
|
||
# measures ~12s live. A persistent backup on $DATA_DIR is kept in sync by every write and gets
|
||
# transparently restored into tmpfs by arr_cache_age_seconds() the moment it notices tmpfs is
|
||
# missing — so a cache that was fresh before a reboot reads as fresh after too.
|
||
#
|
||
# Write-through: any script that already does a live library-list fetch for its own purposes
|
||
# writes the result here as a side effect via arr_cache_write() — no dedicated polling timer
|
||
# needed. arr_cache_write() itself refuses to write while a rescan-type command is active for
|
||
# that arr (2026-07-17) — protects every caller uniformly, not just arr_get_tracked_data().
|
||
# Arrs_Stack/arr_cache_prefill.sh keeps the cache warm two ways: once at array start (10min
|
||
# wait ceiling, closes the cold-boot gap) and again every 30min via CRITICAL_MAINTENANCE_SCRIPTS
|
||
# (1min wait ceiling — a live fetch takes seconds, not the boot-time wait).
|
||
#
|
||
# Consumers should call arr_get_tracked_data() — never read a cache file directly. It handles
|
||
# the fresh/stale-no-rescan/stale-rescan-active branching so no script reimplements it.
|
||
#
|
||
# This is the top-level LIBRARY cache only. A separate, shorter-lived cache exists for the
|
||
# much more expensive per-item track/episode data — see "PER-ITEM TRACKED-FILE CACHE" below.
|
||
# ==============================================================================================
|
||
|
||
# Rescan-type command names per arr — operations long/heavy enough that overlapping with one
|
||
# should trigger a wait rather than a live fetch. Verified live against each arr's API
|
||
# 2026-07-16 (not assumed from memory — RescanSeries/RescanMovie confirmed to scan the whole
|
||
# library when given no id, matching Lidarr's RescanFolders).
|
||
declare -gA ARR_RESCAN_COMMANDS=(
|
||
[lidarr]="RescanFolders DownloadedAlbumsScan RefreshArtist"
|
||
[sonarr]="RescanSeries DownloadedEpisodesScan RefreshSeries"
|
||
[radarr]="RescanMovie DownloadedMoviesScan RefreshMovie"
|
||
)
|
||
|
||
# jq expression computing each arr's "total tracked" number from its library-list response —
|
||
# schemas differ: Lidarr/Sonarr have a per-item file count to sum, Radarr is a binary hasFile
|
||
# per movie (one file at most), so the aggregate has to be a count of true values instead.
|
||
declare -gA ARR_TRACKED_COUNT_EXPR=(
|
||
[lidarr]='[.[].statistics.trackFileCount] | add'
|
||
[sonarr]='[.[].statistics.episodeFileCount] | add'
|
||
[radarr]='[.[] | select(.hasFile==true)] | length'
|
||
)
|
||
|
||
# Library-list endpoint name per arr, for arr_api().
|
||
declare -gA ARR_LIBRARY_ENDPOINT=(
|
||
[lidarr]="artist"
|
||
[sonarr]="series"
|
||
[radarr]="movie"
|
||
)
|
||
|
||
# API version per arr — Lidarr is still v1, Sonarr/Radarr are v3.
|
||
declare -gA ARR_API_VERSION=(
|
||
[lidarr]="v1"
|
||
[sonarr]="v3"
|
||
[radarr]="v3"
|
||
)
|
||
|
||
# tmpfs — primary read/write location. Cleared every reboot; that's fine, a full rebuild for
|
||
# all three arrs measures ~12s live, and arr_cache_age_seconds() transparently restores from
|
||
# the persistent backup below the moment it notices this is missing.
|
||
arr_cache_file() { echo "${ARR_CACHE_DIR}/${1}_tracked_cache.json"; }
|
||
|
||
# Persistent backup — survives reboot. Kept in sync by arr_cache_write() dual-writing here on
|
||
# every write, so it's never more than one write-cycle stale (2026-07-17). Exists purely so a
|
||
# reboot doesn't leave the tmpfs cache genuinely empty until the next live fetch completes —
|
||
# not a source of truth in its own right, just what tmpfs gets restored from when missing.
|
||
arr_cache_backup_file() { echo "${ARR_CACHE_BACKUP_DIR}/${1}_tracked_cache.json"; }
|
||
|
||
arr_rescan_duration_db() { echo "${DATA_DIR}/${1}_rescan_duration.db"; }
|
||
|
||
# Writes the current library-list JSON + computed total tracked count to arr_type's cache.
|
||
# Radarr's movie list alone runs ~16MB — passing that through jq's --argjson as a literal
|
||
# command-line argument blows past the OS's ARG_MAX ("Argument list too long"), the exact
|
||
# same class of bug the queue-pagination fix (2026-07-16, arrs_failed_stalled_recovery.sh)
|
||
# hit before. Fixed the same way: write the payload to a temp file and use --slurpfile,
|
||
# which reads from disk instead of argv.
|
||
#
|
||
# Refuses to write while a rescan-type command is active for arr_type (2026-07-17). Every
|
||
# direct caller here fetches the library list for its own purposes and writes it through as
|
||
# a side effect — none of them go through arr_get_tracked_data()'s fresh/stale/active-rescan
|
||
# branching, so none of them knew to check scan state first. Confirmed live: a caller wrote
|
||
# Lidarr's cache mid-RescanFolders at 22% of the real total, and that partial number then
|
||
# looked exactly like real data loss to every consumer (check_tracked_count_floor et al.)
|
||
# until the scan finished. The guard lives here instead of in each caller so it applies
|
||
# uniformly — last-known-good stays in place until something writes through the real
|
||
# post-scan number (arr_full_rescan.sh does this itself on completion; for a rescan someone
|
||
# else triggered, see arr_rescan_monitor.sh in Tools/).
|
||
# Args: arr_type, items_json (the full library-list response body)
|
||
arr_cache_write() {
|
||
local arr_type="$1" items_json="$2"
|
||
local expr="${ARR_TRACKED_COUNT_EXPR[$arr_type]:-}"
|
||
[[ -z "$expr" ]] && return 1
|
||
|
||
local url_var="${arr_type^^}_URL" key_var="${arr_type^^}_API_KEY"
|
||
local url="${!url_var:-}" api_key="${!key_var:-}" api_version="${ARR_API_VERSION[$arr_type]:-}"
|
||
if [[ -n "$url" && -n "$api_key" && -n "$api_version" ]]; then
|
||
local active_cmd
|
||
active_cmd=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version")
|
||
[[ -n "$active_cmd" ]] && return 1
|
||
fi
|
||
|
||
local total
|
||
total=$(echo "$items_json" | jq "$expr" 2>/dev/null)
|
||
[[ -z "$total" || "$total" == "null" ]] && return 1
|
||
|
||
local cache_file backup_file items_tmp
|
||
cache_file=$(arr_cache_file "$arr_type")
|
||
backup_file=$(arr_cache_backup_file "$arr_type")
|
||
items_tmp=$(mktemp)
|
||
echo "$items_json" > "$items_tmp"
|
||
mkdir -p "$(dirname "$cache_file")" "$(dirname "$backup_file")" 2>/dev/null
|
||
jq -c -n --slurpfile items "$items_tmp" --argjson total "$total" --argjson ts "$(date +%s)" \
|
||
'{ts:$ts, totalTracked:$total, items:$items[0]}' > "${cache_file}.tmp" 2>/dev/null \
|
||
&& mv "${cache_file}.tmp" "$cache_file" \
|
||
&& cp "$cache_file" "$backup_file" 2>/dev/null
|
||
rm -f "$items_tmp"
|
||
}
|
||
|
||
# Echoes the cached library-list JSON array for arr_type if a cache file exists, regardless
|
||
# of age — staleness is arr_get_tracked_data()'s decision, not this raw reader's.
|
||
arr_cache_read_raw() {
|
||
local arr_type="$1"
|
||
local cache_file
|
||
cache_file=$(arr_cache_file "$arr_type")
|
||
[[ -f "$cache_file" ]] || return 1
|
||
jq -c '.items // empty' "$cache_file" 2>/dev/null
|
||
}
|
||
|
||
# Echoes arr_type's cache age in seconds, or a very large number if no cache and no backup
|
||
# exist (so age comparisons naturally treat that as "very stale"). Transparently restores the
|
||
# tmpfs cache from its persistent backup first if tmpfs is missing (e.g. right after a reboot)
|
||
# — a cache that was fresh before reboot reads as fresh here too, not as if it never existed.
|
||
arr_cache_age_seconds() {
|
||
local arr_type="$1"
|
||
local cache_file
|
||
cache_file=$(arr_cache_file "$arr_type")
|
||
if [[ ! -f "$cache_file" ]]; then
|
||
local backup_file
|
||
backup_file=$(arr_cache_backup_file "$arr_type")
|
||
if [[ -f "$backup_file" ]]; then
|
||
mkdir -p "$(dirname "$cache_file")" 2>/dev/null
|
||
cp "$backup_file" "$cache_file" 2>/dev/null
|
||
fi
|
||
fi
|
||
[[ -f "$cache_file" ]] || { echo 999999999; return; }
|
||
local ts
|
||
ts=$(jq -r '.ts // 0' "$cache_file" 2>/dev/null)
|
||
echo $(( $(date +%s) - ${ts:-0} ))
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── PER-ITEM TRACKED-FILE CACHE (Lidarr trackFile, Sonarr episodefile) ───────────────────────
|
||
# ==============================================================================================
|
||
# Different in kind from the tracked-LIBRARY cache above. That one caches the top-level
|
||
# artist/series/movie list (one call). This caches the much more expensive per-item walk —
|
||
# one live API call per artist/series — that lidarr_cleanup.sh/sonarr_cleanup.sh already have
|
||
# to do for their own cleanup decisions regardless. Write-through only, from a walk that's
|
||
# already happening: no independent live-fetch-and-refresh path here, since the field set
|
||
# each consumer needs differs (lidarr_cleanup.sh only needs .path; lidarr_missing_art.sh also
|
||
# needs .albumId) and there's no single generic "fetch everything" call worth centralizing —
|
||
# each script keeps its own live per-item fallback, this is purely a fast path in front of it.
|
||
#
|
||
# Freshness is short (default 4h, not the library cache's 1 day) because this is meant for a
|
||
# script running shortly after in the same maintenance window (e.g. lidarr_missing_art.sh
|
||
# right after lidarr_cleanup.sh), not held across a whole day. tmpfs only, no persistent
|
||
# backup — unlike the library cache, nothing unique lives only here (every consumer already
|
||
# has its own live fallback), and it's short-lived by design, so surviving a reboot doesn't
|
||
# matter the way it did for the always-wanted library cache. (2026-07-17)
|
||
#
|
||
# Future scripts needing this data: call arr_get_cached_items() first, fall back to your own
|
||
# live per-item fetch on a miss, and write through via arr_item_cache_write() if you're the
|
||
# one doing that fetch anyway — same pattern as lidarr_cleanup.sh/sonarr_cleanup.sh below.
|
||
# ==============================================================================================
|
||
|
||
arr_item_cache_file() { echo "${ARR_CACHE_DIR}/${1}_items_cache.json"; }
|
||
|
||
# Args: arr_type, items_json (array of raw per-item track/episode-file objects)
|
||
arr_item_cache_write() {
|
||
local arr_type="$1" items_json="$2"
|
||
|
||
local url_var="${arr_type^^}_URL" key_var="${arr_type^^}_API_KEY"
|
||
local url="${!url_var:-}" api_key="${!key_var:-}" api_version="${ARR_API_VERSION[$arr_type]:-}"
|
||
if [[ -n "$url" && -n "$api_key" && -n "$api_version" ]]; then
|
||
local active_cmd
|
||
active_cmd=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version")
|
||
[[ -n "$active_cmd" ]] && return 1
|
||
fi
|
||
|
||
local cache_file items_tmp
|
||
cache_file=$(arr_item_cache_file "$arr_type")
|
||
items_tmp=$(mktemp)
|
||
echo "$items_json" > "$items_tmp"
|
||
mkdir -p "$(dirname "$cache_file")" 2>/dev/null
|
||
jq -c -n --slurpfile items "$items_tmp" --argjson ts "$(date +%s)" \
|
||
'{ts:$ts, items:$items[0]}' > "${cache_file}.tmp" 2>/dev/null \
|
||
&& mv "${cache_file}.tmp" "$cache_file"
|
||
rm -f "$items_tmp"
|
||
}
|
||
|
||
# Echoes the cached per-item array if present and fresher than max_age_seconds, empty/1 on a
|
||
# miss or stale cache — caller falls back to its own live per-item fetch exactly as it already
|
||
# does today.
|
||
# Args: arr_type, max_age_seconds (default 14400 = 4h)
|
||
arr_get_cached_items() {
|
||
local arr_type="$1" max_age="${2:-14400}"
|
||
local cache_file
|
||
cache_file=$(arr_item_cache_file "$arr_type")
|
||
[[ -f "$cache_file" ]] || return 1
|
||
local ts age
|
||
ts=$(jq -r '.ts // 0' "$cache_file" 2>/dev/null)
|
||
age=$(( $(date +%s) - ${ts:-0} ))
|
||
[[ "$age" -ge "$max_age" ]] && return 1
|
||
jq -c '.items // empty' "$cache_file" 2>/dev/null
|
||
}
|
||
|
||
# Records how long a rescan-type command actually took for arr_type, keyed by command name,
|
||
# so future waits can be calibrated per command type instead of guessed or blended across
|
||
# very different operations — a whole-library RescanFolders/RescanSeries/RescanMovie takes
|
||
# vastly longer than a targeted DownloadedXScan, and averaging them would miscalibrate the
|
||
# wait for both.
|
||
# Args: arr_type, command_name, duration_seconds
|
||
arr_record_rescan_duration() {
|
||
local arr_type="$1" cmd_name="$2" duration="$3"
|
||
[[ -z "$cmd_name" || -z "$duration" ]] && return 1
|
||
local db
|
||
db=$(arr_rescan_duration_db "$arr_type")
|
||
mkdir -p "$(dirname "$db")" 2>/dev/null
|
||
local tmp
|
||
tmp=$(mktemp)
|
||
[[ -f "$db" ]] && grep -v "^${cmd_name}|" "$db" > "$tmp" 2>/dev/null
|
||
echo "${cmd_name}|${duration}" >> "$tmp"
|
||
mv "$tmp" "$db"
|
||
}
|
||
|
||
# Echoes the last recorded duration (seconds) for arr_type + command name, or a fallback
|
||
# default if none has ever been recorded.
|
||
# Args: arr_type, command_name, fallback_default_seconds
|
||
arr_get_rescan_duration() {
|
||
local arr_type="$1" cmd_name="$2" fallback="${3:-300}"
|
||
local db
|
||
db=$(arr_rescan_duration_db "$arr_type")
|
||
[[ -f "$db" ]] || { echo "$fallback"; return; }
|
||
local val
|
||
val=$(grep "^${cmd_name}|" "$db" 2>/dev/null | tail -1 | cut -d'|' -f2)
|
||
echo "${val:-$fallback}"
|
||
}
|
||
|
||
# Checks arr_type's command queue for any currently-active (started or queued) rescan-type
|
||
# command (per ARR_RESCAN_COMMANDS). Echoes the command name of the first one found (so the
|
||
# caller can look up its specific historical duration), empty if none active.
|
||
# Args: arr_type, url, api_key, api_version
|
||
arr_active_rescan_command() {
|
||
local arr_type="$1" url="$2" api_key="$3" api_version="$4"
|
||
local names="${ARR_RESCAN_COMMANDS[$arr_type]:-}"
|
||
[[ -z "$names" ]] && return 1
|
||
local jq_names
|
||
jq_names=$(printf '%s\n' $names | jq -R . | jq -sc .)
|
||
curl -sf --max-time 15 -H "X-Api-Key: $api_key" \
|
||
"${url}/api/${api_version}/command" 2>/dev/null | \
|
||
jq -r --argjson names "$jq_names" '[.[] | select(
|
||
(.status=="started" or .status=="queued") and
|
||
(.name as $n | $names | index($n) != null)
|
||
)] | .[0].name // empty' 2>/dev/null
|
||
}
|
||
|
||
# Waits for an already-active rescan-type command for arr_type to finish — unlike
|
||
# trigger_and_await_command(), this never triggers anything, it only watches. For a rescan
|
||
# someone/something else started (manual intervention, another script) where the caller just
|
||
# needs to know when it's safe to fetch+cache the real post-scan numbers. Returns 0 once no
|
||
# active rescan-type command remains (including immediately if none was active to begin
|
||
# with), 1 on timeout.
|
||
# Args: arr_type, url, api_key, api_version, poll_timeout (default 7200s), poll_interval (default 15s)
|
||
arr_wait_for_active_rescan() {
|
||
local arr_type="$1" url="$2" api_key="$3" api_version="$4"
|
||
local poll_timeout="${5:-7200}" poll_interval="${6:-15}"
|
||
local polled=0 active
|
||
active=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version")
|
||
[[ -z "$active" ]] && return 0
|
||
while [[ "$polled" -lt "$poll_timeout" ]]; do
|
||
sleep "$poll_interval"
|
||
(( polled += poll_interval ))
|
||
active=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version")
|
||
[[ -z "$active" ]] && return 0
|
||
[[ $(( polled % 300 )) -lt "$poll_interval" ]] && log " Still waiting on ${arr_type}'s ${active} (${polled}s elapsed)"
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# Main entry point — the only function consuming scripts should call for tracked library
|
||
# data. Handles fresh/stale-no-rescan/stale-rescan-active branching and always writes through
|
||
# on any live fetch it performs. Echoes the library-list JSON array on success, returns 1 if
|
||
# no usable data (no cache and live fetch impossible) could be obtained.
|
||
# Args: arr_type, url, api_key, api_version, max_age_days (default 1), max_wait_strikes (default 3)
|
||
arr_get_tracked_data() {
|
||
local arr_type="$1" url="$2" api_key="$3" api_version="$4"
|
||
local max_age_days="${5:-${LIDARR_CACHE_MAX_AGE_DAYS:-1}}" max_strikes="${6:-3}"
|
||
local max_age_seconds=$(( max_age_days * 86400 ))
|
||
local age
|
||
age=$(arr_cache_age_seconds "$arr_type")
|
||
|
||
if [[ "$age" -lt "$max_age_seconds" ]]; then
|
||
arr_cache_read_raw "$arr_type" && return 0
|
||
fi
|
||
|
||
# Cache missing or stale — check for an active rescan before deciding how to proceed.
|
||
local active_cmd
|
||
active_cmd=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version")
|
||
|
||
if [[ -n "$active_cmd" ]]; then
|
||
local wait_duration strike arr_label
|
||
wait_duration=$(( $(arr_get_rescan_duration "$arr_type" "$active_cmd" 300) / 2 ))
|
||
[[ "$wait_duration" -lt 30 ]] && wait_duration=30
|
||
arr_label="$(tr '[:lower:]' '[:upper:]' <<< "${arr_type:0:1}")${arr_type:1}"
|
||
for (( strike=1; strike<=max_strikes; strike++ )); do
|
||
# Redirected to stderr — this function's stdout is a data channel (callers
|
||
# capture it via command substitution), never mix log/warn output into it.
|
||
warn " $arr_label busy ($active_cmd) — waiting ${wait_duration}s (strike ${strike}/${max_strikes})" >&2
|
||
sleep "$wait_duration"
|
||
active_cmd=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version")
|
||
[[ -z "$active_cmd" ]] && break
|
||
done
|
||
if [[ -n "$active_cmd" ]]; then
|
||
warn " $arr_label still busy ($active_cmd) after ${max_strikes} strikes — using cache if present, else skipping" >&2
|
||
arr_cache_read_raw "$arr_type" && return 0
|
||
return 1
|
||
fi
|
||
fi
|
||
|
||
# No active rescan (or it just finished mid-wait) — safe to fetch live and refresh cache.
|
||
local endpoint fresh arr_label
|
||
endpoint="${ARR_LIBRARY_ENDPOINT[$arr_type]:-}"
|
||
[[ -z "$endpoint" ]] && return 1
|
||
arr_label="$(tr '[:lower:]' '[:upper:]' <<< "${arr_type:0:1}")${arr_type:1}"
|
||
fresh=$(arr_api "$url" "$api_key" "$api_version" "$endpoint" "$arr_label") || {
|
||
arr_cache_read_raw "$arr_type" && return 0
|
||
return 1
|
||
}
|
||
arr_cache_write "$arr_type" "$fresh"
|
||
echo "$fresh"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── ARR VERSION CHECK ─────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Verifies arr major version matches tested version in master.conf.
|
||
# Exits if version doesn't match — prevents running against untested API structure.
|
||
# Warns and proceeds if version endpoint is unreachable.
|
||
#
|
||
# Usage:
|
||
# check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "Radarr"
|
||
#
|
||
# Arguments:
|
||
# $1 = arr base URL
|
||
# $2 = API key
|
||
# $3 = API path prefix (v3 or v1)
|
||
# $4 = expected major version number from master.conf
|
||
# $5 = arr name for error messages
|
||
|
||
check_arr_version() {
|
||
local url="$1"
|
||
local api_key="$2"
|
||
local api_prefix="$3"
|
||
local expected_major="$4"
|
||
local arr_name="${5:-Arr}"
|
||
|
||
local status_response version major_version
|
||
|
||
status_response=$(curl -sf --max-time 10 \
|
||
-H "X-Api-Key: $api_key" \
|
||
"${url}/api/${api_prefix}/system/status" 2>/dev/null)
|
||
|
||
if [[ -z "$status_response" ]]; then
|
||
warn "$arr_name version check failed — could not reach system/status endpoint"
|
||
warn "Proceeding without version verification — monitor for API errors"
|
||
return 0
|
||
fi
|
||
|
||
version=$(echo "$status_response" | \
|
||
grep -o '"version": *"[^"]*"' | \
|
||
grep -o '[0-9][^"]*' | head -1)
|
||
|
||
if [[ -z "$version" ]]; then
|
||
warn "$arr_name version check failed — could not parse version from response"
|
||
warn "Proceeding without version verification — monitor for API errors"
|
||
return 0
|
||
fi
|
||
|
||
major_version="${version%%.*}"
|
||
|
||
if [[ "$major_version" == "$expected_major" ]]; then
|
||
success "$arr_name version: $version (major $major_version — tested ✅)"
|
||
return 0
|
||
else
|
||
error "$arr_name version mismatch — running v${major_version}, tested against v${expected_major}"
|
||
error "The API endpoint structure may have changed — exiting to protect your library"
|
||
error "Update ${arr_name^^}_VERSION_MAJOR in master.conf after verifying the script works with v${major_version}"
|
||
notify "$arr_name version mismatch on $(hostname) — running v${major_version}, script tested against v${expected_major}" "$arr_name Cleanup" "warning"
|
||
exit 1
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── TMDB DISCOVERY SCORING ────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Shared by playback_aware_radarr_discovery.sh and playback_aware_sonarr_discovery.sh — stage 2
|
||
# candidate scoring against TMDB rating/vote/seed-breadth data. Byte-for-byte identical between
|
||
# the two before this consolidation.
|
||
|
||
# Formats a TMDB vote_average×10 integer back to one decimal place (e.g. 78 → 7.8).
|
||
_fmt_rating() { local v="${1:-0}"; echo "${v::-1}.${v: -1}" 2>/dev/null || echo "$v"; }
|
||
|
||
# Stage 2: TMDB vote_average × 10 as integer (0-40)
|
||
_rating_score_s2() {
|
||
local v="$1"
|
||
if (( v >= 80 )); then echo 40
|
||
elif (( v >= 75 )); then echo 32
|
||
elif (( v >= 70 )); then echo 25
|
||
elif (( v >= 65 )); then echo 18
|
||
elif (( v >= 60 )); then echo 12
|
||
else echo 5
|
||
fi
|
||
}
|
||
|
||
# Stage 2: vote count (0-20)
|
||
_votes_score() {
|
||
local c="$1"
|
||
if (( c >= 10000 )); then echo 20
|
||
elif (( c >= 5000 )); then echo 15
|
||
elif (( c >= 1000 )); then echo 10
|
||
elif (( c >= 200 )); then echo 5
|
||
else echo 2
|
||
fi
|
||
}
|
||
|
||
# Stage 2: seed breadth (0-40)
|
||
_breadth_score() {
|
||
local seeds="$1"
|
||
if (( seeds >= 3 )); then echo 40
|
||
elif (( seeds == 2 )); then echo 25
|
||
else echo 10
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── EMBY LIBRARY SCAN ─────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Triggers Emby "Clean Missing Files" scheduled task after arr cleanup scripts delete files.
|
||
# Without this Emby keeps showing deleted files as ghost entries until its next scheduled scan.
|
||
# User clicks a ghost entry → file not found error. This prevents that entirely.
|
||
#
|
||
# Uses EMBY_URL and EMBY_API_KEY — aliased by detect_hosts() from HOST*_EMBY_* vars.
|
||
# Skips cleanly if Emby is not configured on this host (empty URL or API key).
|
||
#
|
||
# Emby task discovery:
|
||
# GET /ScheduledTasks → find task where Name contains "Clean Missing"
|
||
# POST /ScheduledTasks/Running/{taskId} → trigger it
|
||
#
|
||
# Non-fatal — logs warning if task cannot be triggered but does not exit.
|
||
# Clean Missing Files runs across all libraries at once — one call handles
|
||
# music, TV, and movies together regardless of which arr triggered the cleanup.
|
||
#
|
||
# Usage: notify_emby_scan (no args — always triggers Clean Missing Files)
|
||
|
||
notify_emby_scan() {
|
||
# Skip if Emby not configured on this host
|
||
if [[ -z "${EMBY_URL:-}" ]] || [[ -z "${EMBY_API_KEY:-}" ]]; then
|
||
log "Emby not configured on $MY_ID — skipping library scan notification"
|
||
return 0
|
||
fi
|
||
|
||
log "Notifying Emby to clean missing files..."
|
||
|
||
# Get all scheduled tasks
|
||
local tasks_response
|
||
tasks_response=$(curl -sf --max-time 15 \
|
||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||
"${EMBY_URL}/ScheduledTasks" 2>/dev/null)
|
||
|
||
if [[ -z "$tasks_response" ]]; then
|
||
warn "Could not reach Emby scheduled tasks API at $EMBY_URL — skipping scan"
|
||
return 0
|
||
fi
|
||
|
||
# Find the "Clean Missing Files" task ID
|
||
local task_id
|
||
task_id=$(echo "$tasks_response" | \
|
||
grep -o '"Id":"[^"]*","Name":"[^"]*Clean Missing[^"]*"' | \
|
||
grep -o '"Id":"[^"]*"' | \
|
||
sed 's/"Id":"//;s/"//' | head -1)
|
||
|
||
# Fallback — try "Scan Media Library" if Clean Missing not found
|
||
if [[ -z "$task_id" ]]; then
|
||
task_id=$(echo "$tasks_response" | \
|
||
grep -o '"Id":"[^"]*","Name":"[^"]*Scan Media Library[^"]*"' | \
|
||
grep -o '"Id":"[^"]*"' | \
|
||
sed 's/"Id":"//;s/"//' | head -1)
|
||
[[ -n "$task_id" ]] && log "Clean Missing Files not found — using Scan Media Library"
|
||
fi
|
||
|
||
if [[ -z "$task_id" ]]; then
|
||
warn "Could not find Emby Clean Missing Files or Scan Media Library task"
|
||
warn "Check Emby scheduled tasks — ghost entries will persist until next Emby scan"
|
||
return 0
|
||
fi
|
||
|
||
log "Found Emby task ID: $task_id"
|
||
|
||
# Trigger the task
|
||
local http_code
|
||
http_code=$(curl -sf --max-time 15 \
|
||
-X POST \
|
||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||
-w "%{http_code}" \
|
||
-o /dev/null \
|
||
"${EMBY_URL}/ScheduledTasks/Running/${task_id}" 2>/dev/null)
|
||
|
||
if [[ "$http_code" == "204" ]] || [[ "$http_code" == "200" ]]; then
|
||
warn "$ICON_EMBY Emby Clean Missing Files triggered — ghost entries will be removed"
|
||
else
|
||
warn "Emby task trigger returned HTTP $http_code — ghost entries may persist"
|
||
warn "Check Emby dashboard — manually run Clean Missing Files if needed"
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── API REACHABILITY CHECK ────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Pre-flight check — verifies an API endpoint is reachable before attempting operations.
|
||
# Non-fatal — returns 0 if reachable, 1 if not. Caller decides whether to exit.
|
||
#
|
||
# Usage: check_api "http://localhost:8989" "Sonarr" || exit 1
|
||
|
||
check_api() {
|
||
local url="$1"
|
||
local service="${2:-API}"
|
||
local timeout="${3:-10}"
|
||
|
||
if curl -sf --max-time "$timeout" "$url" >/dev/null 2>&1; then
|
||
log "$service API reachable: $url"
|
||
return 0
|
||
else
|
||
error "$service API not reachable: $url"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── UNRAID VERSION PARITY CHECK ───────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Verifies local and remote servers are running compatible unRAID versions before any
|
||
# remote operation. Version mismatches can mean changed APIs, commands, or behaviours
|
||
# that silently break remote container operations, rsync, or fallback logic.
|
||
#
|
||
# Reads platform OS version on both sides via adapter.
|
||
#
|
||
# Mismatch behaviour (UNRAID_VERSION_MISMATCH_ACTION in master.conf):
|
||
# "warn" — log warning and continue (default for minor/patch differences)
|
||
# "abort" — exit the calling script (default for major version differences)
|
||
# Major version mismatch always aborts regardless of setting
|
||
#
|
||
# Usage: check_os_version_parity || exit 1
|
||
|
||
check_os_version_parity() {
|
||
local local_version remote_version
|
||
|
||
# Read local version
|
||
local_version=$(platform_get_os_version 2>/dev/null)
|
||
if [[ -z "$local_version" ]]; then
|
||
warn "Cannot read local OS version — skipping version parity check"
|
||
return 0
|
||
fi
|
||
|
||
# Read remote version via SSH
|
||
local _probe_cmd
|
||
_probe_cmd=$(platform_os_version_probe_cmd)
|
||
remote_version=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||
"$_probe_cmd" 2>/dev/null)
|
||
if [[ -z "$remote_version" ]]; then
|
||
warn "Cannot read remote OS version from $REMOTE_SERVER_NAME — skipping parity check"
|
||
return 0
|
||
fi
|
||
|
||
log "Version parity: local=$local_version remote=$remote_version"
|
||
|
||
if [[ "$local_version" == "$remote_version" ]]; then
|
||
log "OS versions match: $local_version ✅"
|
||
return 0
|
||
fi
|
||
|
||
# Parse major versions
|
||
local local_major remote_major
|
||
local_major="${local_version%%.*}"
|
||
remote_major="${remote_version%%.*}"
|
||
|
||
if [[ "$local_major" != "$remote_major" ]]; then
|
||
error "OS MAJOR version mismatch — local: $local_version remote: $remote_version"
|
||
error "Major version differences may break remote APIs, commands, and behaviours"
|
||
error "Update both servers to the same major version before running remote operations"
|
||
notify "OS major version mismatch on $(hostname) — local: $local_version remote: $remote_version — remote operations aborted" \
|
||
"Version Parity" "warning"
|
||
return 1
|
||
fi
|
||
|
||
# Minor/patch mismatch — action depends on config
|
||
local action="${UNRAID_VERSION_MISMATCH_ACTION:-warn}"
|
||
warn "OS version mismatch — local: $local_version remote: $remote_version"
|
||
|
||
if [[ "$action" == "abort" ]]; then
|
||
error "UNRAID_VERSION_MISMATCH_ACTION=abort — refusing to continue"
|
||
notify "OS version mismatch on $(hostname) — local: $local_version remote: $remote_version — aborted" \
|
||
"Version Parity" "warning"
|
||
return 1
|
||
fi
|
||
|
||
warn "UNRAID_VERSION_MISMATCH_ACTION=warn — continuing despite mismatch (monitor for issues)"
|
||
return 0
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── REMOTE DOCKER DAEMON CHECK ────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Verifies the remote Docker daemon is responsive before issuing any remote container commands.
|
||
# A hung remote daemon causes docker commands to hang or silently fail — checking first
|
||
# prevents cascading failures and gives clear diagnostic output.
|
||
#
|
||
# Strike system:
|
||
# Each consecutive failed check adds a strike to REMOTE_DOCKER_STRIKES
|
||
# Below REMOTE_DOCKER_STRIKE_LIMIT → warn and return 1 (caller skips operation)
|
||
# At limit → notify critical and return 1 (caller should exit or escalate)
|
||
# Strikes reset when daemon recovers
|
||
#
|
||
# Usage: check_remote_docker_daemon || return 1
|
||
|
||
REMOTE_DOCKER_STRIKES=0
|
||
|
||
check_remote_docker_daemon() {
|
||
local timeout="${1:-10}"
|
||
|
||
if ssh -i "$SSH_KEY" -o ConnectTimeout="$timeout" root@"$REMOTE_SERVER" \
|
||
"timeout $timeout docker info" >/dev/null 2>&1; then
|
||
# Daemon healthy — clear strikes if we had issues
|
||
if [[ "$REMOTE_DOCKER_STRIKES" -gt 0 ]]; then
|
||
info "Remote Docker daemon on $REMOTE_SERVER_NAME recovered — clearing strikes"
|
||
REMOTE_DOCKER_STRIKES=0
|
||
fi
|
||
log "Remote Docker daemon healthy on $REMOTE_SERVER_NAME"
|
||
return 0
|
||
fi
|
||
|
||
REMOTE_DOCKER_STRIKES=$(( REMOTE_DOCKER_STRIKES + 1 ))
|
||
local limit="${REMOTE_DOCKER_STRIKE_LIMIT:-3}"
|
||
warn "Remote Docker daemon not responding on $REMOTE_SERVER_NAME (strike $REMOTE_DOCKER_STRIKES/$limit)"
|
||
|
||
if [[ "$REMOTE_DOCKER_STRIKES" -ge "$limit" ]]; then
|
||
error "Remote Docker daemon unresponsive on $REMOTE_SERVER_NAME after $limit consecutive checks"
|
||
error "Skipping all remote container operations — check Docker on $REMOTE_SERVER_NAME"
|
||
notify "Remote Docker daemon unresponsive on $REMOTE_SERVER_NAME — remote operations skipped on $(hostname)" "Remote Docker Daemon" "warning"
|
||
fi
|
||
|
||
return 1
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── STATUS DISPLAY ────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Prints current runtime configuration — triggered by --status flag in any script.
|
||
# Useful for verifying detect_hosts() resolved correctly and profile loaded as expected.
|
||
|
||
show_status() {
|
||
local local_ver
|
||
local_ver=$(platform_get_os_version 2>/dev/null || echo "unknown")
|
||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||
echo "Local: $LOCAL_SERVER_NAME"
|
||
echo "Remote: $REMOTE_SERVER_NAME"
|
||
echo "IP: ${REMOTE_SERVER:-not resolved}"
|
||
echo "My ID: ${MY_ID:-not set}"
|
||
echo "Remote ID: ${REMOTE_ID:-not set}"
|
||
echo "OS ver: $local_ver"
|
||
echo "Profile: ${PROFILE_NAME:-n/a}"
|
||
|
||
echo "DryRun: ${DRY_RUN:-false}"
|
||
echo "Logging: ${ENABLE_LOGGING:-false}"
|
||
echo "SSH Key: ${SSH_KEY:-not set}"
|
||
echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]:-n/a}"
|
||
echo "Delayed: ${DELAYED_CONTAINERS[*]:-n/a}"
|
||
echo "Excludes: ${EXCLUDE_DIRS[*]:-n/a}"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
}
|
||
|
||
require_partnership() {
|
||
[[ "${PARTNERSHIP_ENABLED:-false}" == "true" ]] && return 0
|
||
log "PARTNERSHIP_ENABLED=false — skipping cross-server operation"
|
||
exit 0
|
||
} |