Files

470 lines
25 KiB
Bash
Executable File

#!/bin/bash
# ==============================================================================================
# ============================= conf_upgrade.sh ================================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Merges a new conf template into an existing user conf while preserving every
# value the user has already set. Run manually when the conf schema changes
# between versions — adds new keys, removes deprecated ones, and keeps the
# structure of the new template exactly.
#
# Keys in template only → ADDED (placeholder/default — user fills in once)
# Keys in target only → REMOVED (deprecated in new version)
# Keys in both → KEPT (target's value always wins, template ignored)
# Comments / blank lines → always from template (structure follows new version)
#
# Supports all conf variable patterns: simple scalars, indexed arrays, and
# associative arrays (declare -A).
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# 1. Validate both --template and --target exist
# 2. Parse each into keys, handling scalars, indexed arrays and declare -A
# 3. Classify every key: ADDED (template only) / REMOVED (target only) / KEPT (both)
# 4. Emit the merged result — template structure, target values — to a temp file
# staged in the target's own directory
# 5. --dry-run stops here and prints the report
# 6. --backup copies the current target to .bak
# 7. Install by atomic rename over the target
#
# Called automatically by git_pull_execute.sh after every pull, for master.conf and this
# host's own host*.conf. The host template is HOSTN_-prefixed and the caller substitutes
# the real MY_ID before merging.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# The User's Value Always Wins
# For any key present in both files, the target's value is kept and the template's is
# discarded. The template supplies structure and new keys, never settings. This is what
# makes the upgrade safe to run unattended after every single pull.
#
# Structure Follows the Template
# Comments, ordering and blank lines come from the template, so an upgraded conf reads
# like the current version rather than accumulating layers of old formatting.
#
# Standalone by Design — Do Not Add load_config.sh
# This script sources nothing. It is the tool that repairs the conf that load_config.sh
# depends on, so it has to work when that conf is broken, partial, or missing keys.
# Sourcing load_config.sh here would make the repair tool fail in exactly the situation
# it exists for. That is also why it uses plain echo instead of log()/error(), and why
# there is no acquire_lock — common.sh is not available to it.
#
# Atomic Install, Never In-Place
# The merged conf is renamed over the target, not copied into it. Every watchdog sources
# load_config.sh on every run; a cp would truncate master.conf and write into it, and
# anything reading during that window gets a partial conf with empty path variables.
#
# Concurrency Handled by Atomicity, Not a Lock
# Two concurrent runs against the same target cannot corrupt it — each stages its own
# temp file and the rename is atomic, so the last writer simply wins. Since the merge is
# idempotent, that outcome is identical to running once.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# The Host Slot Must Be Declared, Never Inferred
# host.conf.template ships HOSTN_ placeholders; a live host conf uses HOST1_ / HOST2_. Key
# matching is literal, so merging the raw template against a real host conf classifies every
# existing key as deprecated — KEPT 0 — and the install drops every credential in the file.
# Verified on HOST1 2026-08-02: it would have removed HOST1_RADARR_API_KEY,
# HOST1_EMBY_API_KEY, HOST1_NPM_PASS and 137 others.
#
# --host-slot HOST<n> is how a caller says which slot the template is being resolved for, and
# it works for any slot — HOST1, HOST2, and whatever a third server would be. The slot is
# still never inferred from the target and applied silently: the caller declares it, and the
# target is read only to contradict a wrong answer. A declared slot that disagrees with the
# target's own keys aborts, because substituting for the wrong slot destroys the file just as
# thoroughly as not substituting at all. Without the flag, a template containing HOSTN is
# refused exactly as before.
#
# Both cases are substituted. HOSTN_ covers the key prefixes, bare HOSTN appears in section
# comments, and lowercase hostn is a real value — the hostn-appdata rsync profile keys. A
# substitution handling only HOSTN_ leaves a conf carrying a profile named hostn-appdata that
# nothing references.
#
# A Target Owning Two Slots Is Refused
# A host conf describes exactly one server. Finding both HOST1_ and HOST2_ key definitions in
# one target means it is not the file it claims to be, so the slot cross-check has nothing
# trustworthy to compare against and the run aborts rather than picking one.
#
# Total Mismatch Is Refused
# Keeping nothing from a populated conf is never a real upgrade; it means the two files do
# not describe the same thing. KEPT 0 with a non-empty REMOVED list aborts. This is the
# general net behind the HOSTN check — the failure mode is silent and total, so it fires in
# --dry-run as well, putting the warning in the report itself.
#
# Root Required to Write
# Installing over the target requires root. --dry-run deliberately does not, so the
# change report can be previewed by anyone.
#
# Atomic Rename
# The temp file is created in the target's own directory so the rename stays within one
# filesystem. On Unraid /tmp is rootfs while the confs live on flash, and a cross-device
# mv degrades to copy-then-unlink — precisely the torn write this avoids.
#
# Permissions Preserved
# mktemp creates 0600; the target's existing mode and owner are copied onto the temp file
# before it is installed, so a conf does not come back with different permissions.
#
# Temp File Cleanup
# An EXIT trap removes the staged file on any early exit, and is cleared once the rename
# has succeeded so the trap cannot delete the installed conf.
#
# Dry-run Mode
# --dry-run prints the full change report (ADDED / REMOVED / KEPT) then exits
# without writing anything. Always preview before applying to production confs.
#
# Backup Option
# --backup writes a .bak copy of the target before overwriting. Use when
# applying to a conf that has never been upgraded before.
#
# File Existence Guards
# Both --template and --target are validated before any parsing begins.
# Missing files abort immediately with a clear error.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# No conf vars. All inputs are CLI flags.
#
# --template <file> New version conf file (source of structure and defaults)
# --target <file> Existing user conf (source of real values — always preserved)
# --host-slot <HOSTn> Resolve HOSTN/hostn placeholders to this slot before merging.
# Required for host.conf.template; meaningless for master.conf.template,
# which has no placeholders. Must match the target's own slot.
# --dry-run Show what would change without writing
# --backup Write a .bak copy of target before modifying
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# conf_upgrade.sh --template Deployment/master.conf.template --target Configurations/master.conf --dry-run
# Preview what would be added, removed, and kept — no changes written.
#
# conf_upgrade.sh --template Deployment/master.conf.template --target Configurations/master.conf --backup
# Apply the upgrade, writing a .bak first.
#
# conf_upgrade.sh --template Deployment/master.conf.template --target Configurations/master.conf
# Apply the upgrade in-place with no backup.
#
# conf_upgrade.sh --template Deployment/host.conf.template --target Configurations/host1.conf \
# --host-slot HOST1 --dry-run
# Preview a host conf upgrade. HOSTN/hostn are resolved to HOST1/host1 first. Swap in HOST2
# and host2.conf for the other server — the template is the same file for every slot.
#
# ==============================================================================================
set -uo pipefail
# ── Arguments ────────────────────────────────────────────────────────────────────────────────
TEMPLATE=""
TARGET=""
DRY_RUN=false
BACKUP=false
HOST_SLOT=""
_RESOLVED_TMPL="" # set only when --host-slot triggers a substitution; cleaned on exit
while [[ $# -gt 0 ]]; do
case "$1" in
--template) TEMPLATE="$2"; shift 2 ;;
--target) TARGET="$2"; shift 2 ;;
--host-slot) HOST_SLOT="$2"; shift 2 ;;
--dry-run) DRY_RUN=true; shift ;;
--backup) BACKUP=true; shift ;;
*) echo "Unknown option: $1" >&2; exit 1 ;;
esac
done
[[ -z "$TEMPLATE" ]] && { echo "Error: --template required" >&2; exit 1; }
[[ -z "$TARGET" ]] && { echo "Error: --target required" >&2; exit 1; }
[[ -f "$TEMPLATE" ]] || { echo "Error: template not found: $TEMPLATE" >&2; exit 1; }
[[ -f "$TARGET" ]] || { echo "Error: target not found: $TARGET" >&2; exit 1; }
# ── Host slot resolution ─────────────────────────────────────────────────────────────────────
#
# host.conf.template ships HOSTN_ placeholders; a live host conf uses HOST1_ / HOST2_. Key
# matching below is literal, so merging the raw template against a real host conf classifies
# EVERY existing key as deprecated and every template key as new — KEPT 0, and the install
# would drop every credential in the file.
#
# --host-slot is how a caller declares which slot the template is for. The slot is never
# inferred from the target and silently applied: the caller states it, and the target is used
# only to contradict a wrong answer. Substituting for the wrong slot is the same catastrophe as
# not substituting at all, so a declared slot that disagrees with the target is refused.
#
# Both cases matter. HOSTN_ covers the 159 key prefixes; bare HOSTN appears in section comments,
# and lowercase hostn is a real value — the hostn-appdata rsync profile keys. A substitution
# that only handles HOSTN_ leaves a live conf with a profile named hostn-appdata that nothing
# references, which is what the pull script did before this flag existed.
_target_slots=$(grep -oE '^[[:space:]]*HOST[0-9]+_' "$TARGET" 2>/dev/null \
| grep -oE 'HOST[0-9]+' | sort -u)
_target_slot=$(echo "$_target_slots" | head -1)
if [[ $(echo "$_target_slots" | grep -c .) -gt 1 ]]; then
echo "Error: '$TARGET' defines keys for more than one host slot:" >&2
echo " $(echo "$_target_slots" | tr '\n' ' ')" >&2
echo " A host conf owns exactly one slot. Refusing rather than picking one." >&2
exit 1
fi
if [[ -n "$HOST_SLOT" ]]; then
if ! [[ "$HOST_SLOT" =~ ^HOST[0-9]+$ ]]; then
echo "Error: --host-slot must be HOST<n> (e.g. HOST1, HOST2) — got '$HOST_SLOT'" >&2
exit 1
fi
if [[ -n "$_target_slot" && "$_target_slot" != "$HOST_SLOT" ]]; then
echo "Error: --host-slot says $HOST_SLOT but '$TARGET' defines ${_target_slot}_ keys." >&2
echo " Substituting for the wrong slot removes every key the target actually has," >&2
echo " credentials included. Refusing." >&2
exit 1
fi
if grep -q -e 'HOSTN' -e 'hostn' "$TEMPLATE" 2>/dev/null; then
_lower=$(echo "$HOST_SLOT" | tr '[:upper:]' '[:lower:]')
_RESOLVED_TMPL="$(mktemp)"
trap '[[ -n "${_RESOLVED_TMPL:-}" ]] && rm -f "$_RESOLVED_TMPL"' EXIT
sed -e "s/HOSTN/${HOST_SLOT}/g" -e "s/hostn/${_lower}/g" "$TEMPLATE" > "$_RESOLVED_TMPL"
TEMPLATE="$_RESOLVED_TMPL"
echo " Resolved HOSTN → ${HOST_SLOT} for this host"
fi
elif grep -q 'HOSTN' "$TEMPLATE" 2>/dev/null; then
# No slot declared and the template is still generic — the original refusal, unchanged.
if [[ -n "$_target_slot" ]]; then
_lower=$(echo "$_target_slot" | tr '[:upper:]' '[:lower:]')
echo "Error: template still contains HOSTN placeholders, but the target uses ${_target_slot}_." >&2
echo " Merging as-is would classify all ${_target_slot}_ keys as deprecated and remove" >&2
echo " them — including every credential. Declare the slot:" >&2
echo "" >&2
echo " $0 --template $TEMPLATE --target $TARGET --host-slot ${_target_slot} --dry-run" >&2
echo "" >&2
exit 1
fi
fi
# ── Block-parsing patterns ───────────────────────────────────────────────────────────────────
# All three walkers below share these. A single-line array — KEY=(a b c) — both opens and
# closes on one line. Matching only the opener leaves the walker inside a block it never
# leaves, so every key until the next standalone ")" becomes invisible: skipped by
# _collect_stats, dropped from HOST_MAP by _parse_target, and emitted from the template
# instead of the user's conf by _write_merged. _ARR_ONELINE_RE is what stops that.
_ARR_DECL_RE='^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\('
_ARR_OPEN_RE='^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\('
_ARR_CLOSE_RE='^[[:space:]]*\)[[:space:]]*(#.*)?$'
# The [^#]* is deliberate: it forces the closing ")" to appear before any comment, so an
# opener like FOO=( # see note (here) is not mistaken for a complete single-line array.
_ARR_ONELINE_RE='=\([^#]*\)[[:space:]]*(#.*)?$'
_SCALAR_RE='^[[:space:]]*([A-Z0-9_]+)[[:space:]]*='
# ── Parse target → KEY → full definition block ───────────────────────────────────────────────
declare -A HOST_MAP # KEY → complete definition line(s) from user's conf
_parse_target() {
local in_block=false cur_key="" cur_block="" line
while IFS= read -r line || [[ -n "$line" ]]; do
if [[ "$in_block" == true ]]; then
cur_block+="$line"$'\n'
# Closing ) — optional trailing whitespace and comment
if [[ "$line" =~ $_ARR_CLOSE_RE ]]; then
HOST_MAP["$cur_key"]="$cur_block"
in_block=false; cur_key=""; cur_block=""
fi
else
# declare -A KEY=( / KEY=(
if [[ "$line" =~ $_ARR_DECL_RE ]] || [[ "$line" =~ $_ARR_OPEN_RE ]]; then
cur_key="${BASH_REMATCH[1]}"
# KEY=(a b c) — opens and closes on one line, never enter block mode
if [[ "$line" =~ $_ARR_ONELINE_RE ]]; then
HOST_MAP["$cur_key"]="$line"$'\n'
cur_key=""
else
in_block=true; cur_block="$line"$'\n'
fi
# KEY=value (simple scalar)
elif [[ "$line" =~ $_SCALAR_RE ]]; then
HOST_MAP["${BASH_REMATCH[1]}"]="$line"$'\n'
fi
fi
done < "$TARGET"
}
# ── Walk template — collect stats (must run in current shell so arrays persist) ──────────────
declare -a ADDED=() KEPT=() REMOVED=()
declare -A TMPL_SEEN=()
_collect_stats() {
local in_block=false cur_key="" line
while IFS= read -r line || [[ -n "$line" ]]; do
if [[ "$in_block" == true ]]; then
if [[ "$line" =~ $_ARR_CLOSE_RE ]]; then
in_block=false
TMPL_SEEN["$cur_key"]=1
if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then KEPT+=("$cur_key")
else ADDED+=("$cur_key"); fi
cur_key=""
fi
else
if [[ "$line" =~ $_ARR_DECL_RE ]] || [[ "$line" =~ $_ARR_OPEN_RE ]]; then
cur_key="${BASH_REMATCH[1]}"
if [[ "$line" =~ $_ARR_ONELINE_RE ]]; then
TMPL_SEEN["$cur_key"]=1
if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then KEPT+=("$cur_key")
else ADDED+=("$cur_key"); fi
cur_key=""
else
in_block=true
fi
elif [[ "$line" =~ $_SCALAR_RE ]]; then
local k="${BASH_REMATCH[1]}"
TMPL_SEEN["$k"]=1
if [[ -n "${HOST_MAP[$k]+_}" ]]; then KEPT+=("$k")
else ADDED+=("$k"); fi
fi
fi
done < "$TEMPLATE"
for key in "${!HOST_MAP[@]}"; do
[[ -z "${TMPL_SEEN[$key]+_}" ]] && REMOVED+=("$key")
done
}
# ── Walk template — write merged output ──────────────────────────────────────────────────────
# Runs in a subshell (stdout redirected) — array mutations are intentionally discarded here.
_write_merged() {
local in_block=false cur_key="" cur_block="" line
while IFS= read -r line || [[ -n "$line" ]]; do
if [[ "$in_block" == true ]]; then
cur_block+="$line"$'\n'
if [[ "$line" =~ $_ARR_CLOSE_RE ]]; then
in_block=false
if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then printf '%s' "${HOST_MAP[$cur_key]}"
else printf '%s' "$cur_block"; fi
cur_key=""; cur_block=""
fi
else
if [[ "$line" =~ $_ARR_DECL_RE ]] || [[ "$line" =~ $_ARR_OPEN_RE ]]; then
cur_key="${BASH_REMATCH[1]}"
if [[ "$line" =~ $_ARR_ONELINE_RE ]]; then
if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then printf '%s' "${HOST_MAP[$cur_key]}"
else printf '%s\n' "$line"; fi
cur_key=""
else
in_block=true; cur_block="$line"$'\n'
fi
elif [[ "$line" =~ $_SCALAR_RE ]]; then
local k="${BASH_REMATCH[1]}"
if [[ -n "${HOST_MAP[$k]+_}" ]]; then printf '%s' "${HOST_MAP[$k]}"
else printf '%s\n' "$line"; fi
else
printf '%s\n' "$line"
fi
fi
done < "$TEMPLATE"
}
# ── Run ───────────────────────────────────────────────────────────────────────────────────────
_parse_target
_collect_stats
# ── Report ────────────────────────────────────────────────────────────────────────────────────
TARGET_NAME="$(basename "$TARGET")"
echo ""
echo "── conf_upgrade: $TARGET_NAME ──────────────────────────────────────────"
if [[ ${#ADDED[@]} -gt 0 ]]; then
echo " ADDED (new — fill in your values where needed):"
for k in "${ADDED[@]}"; do echo " + $k"; done
fi
if [[ ${#REMOVED[@]} -gt 0 ]]; then
echo " REMOVED (deprecated — no longer in this version):"
for k in "${REMOVED[@]}"; do echo " - $k"; done
fi
echo " KEPT ${#KEPT[@]} existing vars — your values preserved"
# ── Guard: total mismatch ────────────────────────────────────────────────────────────────────
#
# Keeping nothing from a populated conf is never a real upgrade — it is the signature of the
# two files not describing the same thing (wrong template, wrong target, unsubstituted
# placeholders). The HOSTN check above catches the known cause; this catches the rest, because
# the failure mode is silent and total: every value in the file is replaced by a default.
# Deliberately fires in --dry-run too, so the report itself carries the warning.
if [[ ${#KEPT[@]} -eq 0 && ${#REMOVED[@]} -gt 0 ]]; then
echo "────────────────────────────────────────────────────────────────────────"
echo ""
echo "Error: refusing — this would keep NOTHING and remove all ${#REMOVED[@]} existing keys." >&2
echo " A genuine upgrade preserves values; keeping zero means the template and the" >&2
echo " target do not describe the same conf. Check that --template matches --target" >&2
echo " and that any HOSTN placeholders were substituted for this host's slot." >&2
echo "" >&2
exit 1
fi
if [[ ${#ADDED[@]} -eq 0 && ${#REMOVED[@]} -eq 0 ]]; then
echo " Already up to date — no changes needed."
echo "────────────────────────────────────────────────────────────────────────"
echo ""
exit 0
fi
echo "────────────────────────────────────────────────────────────────────────"
echo ""
# ── Apply ─────────────────────────────────────────────────────────────────────────────────────
if [[ "$DRY_RUN" == true ]]; then
echo "(dry-run — no changes written)"
exit 0
fi
# Checked here rather than at the top: --dry-run is a read-only report and is useful to
# anyone, but installing over a conf under /boot needs root. Plain echo because this script
# deliberately does not source common.sh — see the header.
if [[ "$EUID" -ne 0 ]]; then
echo "ERROR: writing '$TARGET' requires root (use --dry-run to preview as any user)" >&2
exit 1
fi
# Staged beside the target, not in /tmp. mv is only atomic within one filesystem, and on
# Unraid /tmp is rootfs while the confs live on flash — a cross-device mv silently degrades
# to copy-then-unlink, which is exactly the torn write this is meant to prevent.
TMPOUT="$(mktemp "${TARGET}.XXXXXX")"
trap 'rm -f "$TMPOUT"; [[ -n "${_RESOLVED_TMPL:-}" ]] && rm -f "$_RESOLVED_TMPL"' EXIT
# mktemp creates 0600; carry the target's existing mode/owner across so the installed conf
# does not come back with different permissions than it went in with.
chmod --reference="$TARGET" "$TMPOUT" 2>/dev/null || true
chown --reference="$TARGET" "$TMPOUT" 2>/dev/null || true
_write_merged > "$TMPOUT"
if [[ "$BACKUP" == true ]]; then
cp -a "$TARGET" "${TARGET}.bak"
echo "Backup: ${TARGET}.bak"
fi
# Atomic install. cp would truncate the live conf and write into it, leaving a window where
# anything sourcing load_config.sh reads a half-written master.conf — every watchdog does
# that constantly. A rename swaps the inode: readers get the old file or the new one.
mv -f "$TMPOUT" "$TARGET"
trap - EXIT
[[ -n "$_RESOLVED_TMPL" ]] && rm -f "$_RESOLVED_TMPL"
echo "Updated: $TARGET"