diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index ae0900d..b4dc5e6 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -503,6 +503,11 @@ "Arrs_Stack/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards) "Arrs_Stack/radarr_tmdb_removed.sh" # remove movies dropped from TMDb "Arrs_Stack/sonarr_tvdb_removed.sh" # remove series dropped from TVDB + # Before the two docker jobs below, deliberately: they pull images and restart + # containers, so a container can legitimately not exist for a moment while they run. + # Reading docker in its settled overnight state keeps a rebuild from costing a strike. + # Three strikes at this cadence means three days genuinely absent before conf is edited. + "Tools/conf_container_prune.sh" # drop containers that no longer exist from the conf lists "Docker_Essentials/docker_update.sh" # pull container image updates before restart "Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last # Last, and after git_pull_execute.sh which must always run first. Daily rather than with @@ -696,6 +701,10 @@ RSYNC_MIN_FREE_GB=50 # warn if any disk backing the remote share is under this # (0 disables). Online is not writable: a full destination # makes rsync stall rather than fail. Warns, never aborts. + CONF_PRUNE_STRIKE_LIMIT=3 # consecutive runs a container must be absent from docker + # before its name is removed from the conf lists. A + # rebuild makes a container vanish briefly; one miss + # must never be enough to edit conf. CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override CONTAINER_DELAY=5 # seconds before starting delayed containers diff --git a/Tools/conf_container_prune.sh b/Tools/conf_container_prune.sh new file mode 100755 index 0000000..f2ffbec --- /dev/null +++ b/Tools/conf_container_prune.sh @@ -0,0 +1,220 @@ +#!/bin/bash +# ══════════════════════════════════════════════════════════════════════════════════════════════ +# PURPOSE +# Keep the conf's container lists honest. A container removed from docker leaves its name behind +# in every list that referenced it, and those names go on being acted upon — a watchdog looking +# for a required container that no longer exists, a fallback tier promising to start one, a +# restart job reaching for a name docker has never heard of. +# +# OPERATIONAL MODEL +# Strikes, never a single miss. A container is absent from `docker ps -a` for entirely ordinary +# reasons: it is mid-rebuild, its image is being pulled, the operator is editing it. Five +# containers on this host were briefly absent on 2026-08-23 while their templates were rewritten, +# and a prune on first sight would have stripped all five from conf. A name must be missing on +# CONF_PRUNE_STRIKE_LIMIT consecutive runs before anything is written. +# +# Seeing a container resets its strike to zero immediately, so a rebuild costs one strike at most. +# +# DESIGN PRINCIPLES +# An explicit allow list of keys, never a pattern. +# Not every list of container names describes THIS host. CRITICAL_CONTAINER_NAMES and +# DELAYED_CONTAINERS are the containers stopped on the REMOTE before an rsync — pruning those +# against local docker would empty them on a host that legitimately runs none of them, and +# rsync would then stop nothing before copying a live database. They are excluded by name and +# must stay excluded. +# +# A docker that cannot be read is not a docker with nothing in it. +# If `docker ps -a` fails, times out, or returns nothing on a host that is meant to run +# containers, EVERY name looks missing and one run would strike the entire configuration. +# That case aborts before a single strike is recorded. +# +# The conf is backed up and re-read before the change is accepted. +# An array can be written as KEY=(a b c) or spread over many lines, and this project has +# already lost 68 vars to an editor that only understood one of those shapes. The rewrite is +# verified by sourcing the result in a subshell and confirming the key still parses as an +# array with exactly one fewer element. +# +# USAGE +# conf_container_prune.sh strike, and prune anything at the limit +# conf_container_prune.sh --dry-run report what would be struck and pruned, write nothing +# conf_container_prune.sh --status show current strikes and stop +# +# DEPENDS ON +# CONF_PRUNE_STRIKE_LIMIT consecutive misses before a name is removed (master.conf, default 3) +# STATE_DIR/conf_prune_strikes.db the strike counts, one per container +# ══════════════════════════════════════════════════════════════════════════════════════════════ + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../load_config.sh" + +DRY_RUN="${DRY_RUN:-false}" +MODE="run" +for a in "$@"; do + case "$a" in + --dry-run) DRY_RUN=true ;; + --status) MODE="status" ;; + esac +done + +detect_hosts +LIMIT="${CONF_PRUNE_STRIKE_LIMIT:-3}" +STRIKE_FILE="${STATE_DIR}/conf_prune_strikes.db" +[[ -f "$STRIKE_FILE" ]] || : > "$STRIKE_FILE" + +# ── Which keys describe containers on THIS host ─────────────────────────────────────────────── +# Deliberately enumerated. CRITICAL_CONTAINER_NAMES and DELAYED_CONTAINERS are absent on purpose — +# see the design note above; they name containers on the REMOTE. +PRUNE_KEYS=( + "FALLBACK_${MY_ID}_TIER1" "FALLBACK_${MY_ID}_TIER2" + "FALLBACK_${MY_ID}_TIER3" "FALLBACK_${MY_ID}_TIER4" + "${MY_ID}_WATCHDOG_REQUIRED_CONTAINERS" + "${MY_ID}_DAILY_RESTART_CONTAINERS" + "${MY_ID}_WEEKLY_RESTART_CONTAINERS" + "${MY_ID}_DDNS_CONTAINERS" + "${MY_ID}_NETWORK_CONNECT_CONTAINERS" + "${MY_ID}_RW_PAUSE_CONTAINERS" + "${MY_ID}_RW_STOP_CONTAINERS" + "${MY_ID}_PARTNERSHIP_OWN_CONTAINERS" + "${MY_ID}_PARTNERSHIP_REPLACE_CONTAINERS" + "${MY_ID}_PARTNERSHIP_ARR_REPLACE_CONTAINERS" + "RW_CRITICAL_CONTAINERS" +) +CONF_FILES=("$CONF_DIR/master.conf" "$CONF_DIR/$(echo "$MY_ID" | tr '[:upper:]' '[:lower:]').conf") + +# ── What docker actually has ────────────────────────────────────────────────────────────────── +DOCKER_NAMES=$(timeout "${DOCKER_TIMEOUT:-30}" docker ps -a --format '{{.Names}}' 2>/dev/null) +DOCKER_RC=$? +DOCKER_COUNT=$(printf '%s\n' "$DOCKER_NAMES" | grep -c .) + +if [[ "$DOCKER_RC" -ne 0 ]] || [[ "$DOCKER_COUNT" -eq 0 ]]; then + error "$ICON_DOCKER docker returned $DOCKER_COUNT container(s) (rc=$DOCKER_RC) — refusing to strike anything" + error " Every configured name would look missing, and one run would empty the lists." + exit 1 +fi +log "$ICON_DOCKER docker reports $DOCKER_COUNT container(s); strike limit $LIMIT" + +container_exists() { printf '%s\n' "$DOCKER_NAMES" | grep -qxF "$1"; } + +# Names currently in the conf, per key, deduped for reporting. +declare -A SEEN_IN_KEYS=() +ALL_NAMES=() +for f in "${CONF_FILES[@]}"; do + [[ -f "$f" ]] || continue + for k in "${PRUNE_KEYS[@]}"; do + # Both array shapes: KEY=(a b c) on one line, or opened and closed across many. + vals=$(awk -v key="$k" ' + $0 ~ "^[[:space:]]*"key"=\\(" { inside=1 } + inside { print } + inside && /\)/ && $0 !~ "^[[:space:]]*"key"=\\($" { inside=0 } + ' "$f" | grep -oE '"[^"]+"' | tr -d '"') + for v in $vals; do + [[ -z "$v" ]] && continue + ALL_NAMES+=("$v") + SEEN_IN_KEYS["$v"]+="$(basename "$f"):$k " + done + done +done + +if [[ ${#ALL_NAMES[@]} -eq 0 ]]; then + log "$ICON_DONE No container names in any prunable list — nothing to do" + exit 0 +fi + +if [[ "$MODE" == "status" ]]; then + echo "Strikes (limit $LIMIT):" + while IFS=: read -r name count; do + [[ -z "$name" ]] && continue + printf ' %-30s %s\n' "$name" "$count" + done < "$STRIKE_FILE" + exit 0 +fi + +# ── Strike, or clear ────────────────────────────────────────────────────────────────────────── +PRUNE=() +declare -A DONE=() +for name in "${ALL_NAMES[@]}"; do + [[ -n "${DONE[$name]:-}" ]] && continue + DONE["$name"]=1 + + if container_exists "$name"; then + prev=$(wd_state_get "$name" "$STRIKE_FILE") + if [[ -n "$prev" && "$prev" != "0" ]]; then + log " $name is back — clearing $prev strike(s)" + [[ "$DRY_RUN" == false ]] && wd_state_set "$name" 0 "$STRIKE_FILE" + fi + continue + fi + + prev=$(wd_state_get "$name" "$STRIKE_FILE"); prev="${prev//[^0-9]/}"; prev="${prev:-0}" + # Pre-increment, never (( n++ )) — that returns 1 when n is 0 and would trip any errexit above. + next=$(( prev + 1 )) + if [[ "$next" -ge "$LIMIT" ]]; then + warn " $name missing from docker — strike $next/$LIMIT, PRUNING" + PRUNE+=("$name") + else + warn " $name missing from docker — strike $next/$LIMIT" + [[ "$DRY_RUN" == false ]] && wd_state_set "$name" "$next" "$STRIKE_FILE" + fi +done + +if [[ ${#PRUNE[@]} -eq 0 ]]; then + log "$ICON_DONE Nothing at the strike limit" + exit 0 +fi + +# ── Remove, one name at a time, verifying after each file ───────────────────────────────────── +remove_from_conf() { + local file="$1" key="$2" name="$3" + local tmp; tmp=$(mktemp) + awk -v key="$key" -v name="\"$name\"" ' + $0 ~ "^[[:space:]]*"key"=\\(" { inside=1 } + { + if (inside) { + # Single-line form: drop just the one quoted entry, keep the rest of the line. + if ($0 ~ /\)[[:space:]]*(#.*)?$/ && $0 ~ "^[[:space:]]*"key"=\\(") { + gsub(name"[[:space:]]*", "") + print; inside=0; next + } + # Multi-line form: a line that is only this entry disappears entirely. + line=$0; gsub(/^[[:space:]]+|[[:space:]]+$/, "", line) + if (line == name) next + if ($0 ~ /^[[:space:]]*\)/) inside=0 + } + print + } + ' "$file" > "$tmp" || { rm -f "$tmp"; return 1; } + + # Accept only if the file still sources and the key still parses as an array. + if ! ( set +u; source "$tmp" >/dev/null 2>&1 ); then + rm -f "$tmp"; error " rewrite of $(basename "$file") would not source — kept the original" + return 1 + fi + mv "$tmp" "$file" + return 0 +} + +for name in "${PRUNE[@]}"; do + for f in "${CONF_FILES[@]}"; do + [[ -f "$f" ]] || continue + for k in "${PRUNE_KEYS[@]}"; do + grep -qE "^[[:space:]]*$k=\(" "$f" || continue + grep -qF "\"$name\"" "$f" || continue + if [[ "$DRY_RUN" == true ]]; then + warn " DRY RUN — would remove $name from $k in $(basename "$f")" + continue + fi + cp "$f" "$f.bak-prune-$(date +%Y%m%d-%H%M%S)" + if remove_from_conf "$f" "$k" "$name"; then + log " removed $name from $k in $(basename "$f")" + fi + done + done + [[ "$DRY_RUN" == false ]] && wd_state_set "$name" 0 "$STRIKE_FILE" +done + +if [[ "$DRY_RUN" == false ]]; then + notify "Pruned ${#PRUNE[@]} removed container(s) from conf on $(hostname) ($MY_ID): ${PRUNE[*]}" \ + "Conf Container Prune" "normal" +fi +log "$ICON_DONE Pruned ${#PRUNE[@]} container(s): ${PRUNE[*]}" +exit 0