239 lines
11 KiB
Bash
Executable File
239 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
|
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
|
# PURPOSE
|
|
# Keep the conf's container lists honest. A container removed from docker leaves its name behind
|
|
# in every list that referenced it, and those names go on being acted upon — a watchdog looking
|
|
# for a required container that no longer exists, a fallback tier promising to start one, a
|
|
# restart job reaching for a name docker has never heard of.
|
|
#
|
|
# OPERATIONAL MODEL
|
|
# Strikes, never a single miss. A container is absent from `docker ps -a` for entirely ordinary
|
|
# reasons: it is mid-rebuild, its image is being pulled, the operator is editing it. Five
|
|
# containers on this host were briefly absent on 2026-08-23 while their templates were rewritten,
|
|
# and a prune on first sight would have stripped all five from conf. A name must be missing on
|
|
# CONF_PRUNE_STRIKE_LIMIT consecutive runs before anything is written.
|
|
#
|
|
# Seeing a container resets its strike to zero immediately, so a rebuild costs one strike at most.
|
|
#
|
|
# DESIGN PRINCIPLES
|
|
# An explicit allow list of keys, never a pattern.
|
|
# Not every list of container names describes THIS host. CRITICAL_CONTAINER_NAMES and
|
|
# DELAYED_CONTAINERS are the containers stopped on the REMOTE before an rsync — pruning those
|
|
# against local docker would empty them on a host that legitimately runs none of them, and
|
|
# rsync would then stop nothing before copying a live database. They are excluded by name and
|
|
# must stay excluded.
|
|
#
|
|
# OPERATIONAL SAFEGUARDS
|
|
# A docker that cannot be read is not a docker with nothing in it.
|
|
# If `docker ps -a` fails, times out, or returns nothing on a host that is meant to run
|
|
# containers, EVERY name looks missing and one run would strike the entire configuration.
|
|
# That case aborts before a single strike is recorded.
|
|
#
|
|
# The conf is backed up and re-read before the change is accepted.
|
|
# An array can be written as KEY=(a b c) or spread over many lines, and this project has
|
|
# already lost 68 vars to an editor that only understood one of those shapes. The rewrite is
|
|
# verified by sourcing the result in a subshell and confirming the key still parses as an
|
|
# array with exactly one fewer element.
|
|
#
|
|
# Each removal is its own verified rewrite.
|
|
# Several names can reach the limit in one run, and each is removed and re-verified
|
|
# independently rather than batched into a single edit. A rewrite that fails verification
|
|
# therefore costs that one entry, not every entry the run intended to prune.
|
|
#
|
|
# --dry-run records no strike. A dry run that advanced the counter would eventually prune
|
|
# through repetition alone, which is the opposite of what it is for.
|
|
#
|
|
# CONFIGURATION
|
|
# master.conf
|
|
# CONF_PRUNE_STRIKE_LIMIT consecutive runs a name must be missing before it is removed.
|
|
# Seeing the container again resets its strike to zero immediately,
|
|
# so a rebuild costs one strike at most.
|
|
#
|
|
# The keys this may prune are an explicit allow list in the script, deliberately not a conf
|
|
# value — see DESIGN PRINCIPLES for why a pattern is the wrong shape here.
|
|
#
|
|
# RUNTIME MODES
|
|
# conf_container_prune.sh strike, and prune anything at the limit
|
|
# conf_container_prune.sh --dry-run report what would be struck and pruned, write nothing
|
|
# conf_container_prune.sh --status show current strikes and stop
|
|
#
|
|
# DEPENDS ON
|
|
# CONF_PRUNE_STRIKE_LIMIT consecutive misses before a name is removed (master.conf, default 3)
|
|
# STATE_DIR/conf_prune_strikes.db the strike counts, one per container
|
|
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
DRY_RUN="${DRY_RUN:-false}"
|
|
MODE="run"
|
|
for a in "$@"; do
|
|
case "$a" in
|
|
--dry-run) DRY_RUN=true ;;
|
|
--status) MODE="status" ;;
|
|
esac
|
|
done
|
|
|
|
detect_hosts
|
|
LIMIT="${CONF_PRUNE_STRIKE_LIMIT:-3}"
|
|
STRIKE_FILE="${STATE_DIR}/conf_prune_strikes.db"
|
|
[[ -f "$STRIKE_FILE" ]] || : > "$STRIKE_FILE"
|
|
|
|
# ── Which keys describe containers on THIS host ───────────────────────────────────────────────
|
|
# Deliberately enumerated. CRITICAL_CONTAINER_NAMES and DELAYED_CONTAINERS are absent on purpose —
|
|
# see the design note above; they name containers on the REMOTE.
|
|
PRUNE_KEYS=(
|
|
"FALLBACK_${MY_ID}_TIER1" "FALLBACK_${MY_ID}_TIER2"
|
|
"FALLBACK_${MY_ID}_TIER3" "FALLBACK_${MY_ID}_TIER4"
|
|
"${MY_ID}_WATCHDOG_REQUIRED_CONTAINERS"
|
|
"${MY_ID}_DAILY_RESTART_CONTAINERS"
|
|
"${MY_ID}_WEEKLY_RESTART_CONTAINERS"
|
|
"${MY_ID}_DDNS_CONTAINERS"
|
|
"${MY_ID}_NETWORK_CONNECT_CONTAINERS"
|
|
"${MY_ID}_RW_PAUSE_CONTAINERS"
|
|
"${MY_ID}_RW_STOP_CONTAINERS"
|
|
"${MY_ID}_PARTNERSHIP_OWN_CONTAINERS"
|
|
"${MY_ID}_PARTNERSHIP_REPLACE_CONTAINERS"
|
|
"${MY_ID}_PARTNERSHIP_ARR_REPLACE_CONTAINERS"
|
|
"RW_CRITICAL_CONTAINERS"
|
|
)
|
|
CONF_FILES=("$CONF_DIR/master.conf" "$CONF_DIR/$(echo "$MY_ID" | tr '[:upper:]' '[:lower:]').conf")
|
|
|
|
# ── What docker actually has ──────────────────────────────────────────────────────────────────
|
|
DOCKER_NAMES=$(timeout "${DOCKER_TIMEOUT:-30}" docker ps -a --format '{{.Names}}' 2>/dev/null)
|
|
DOCKER_RC=$?
|
|
DOCKER_COUNT=$(printf '%s\n' "$DOCKER_NAMES" | grep -c .)
|
|
|
|
if [[ "$DOCKER_RC" -ne 0 ]] || [[ "$DOCKER_COUNT" -eq 0 ]]; then
|
|
error "$ICON_DOCKER docker returned $DOCKER_COUNT container(s) (rc=$DOCKER_RC) — refusing to strike anything"
|
|
error " Every configured name would look missing, and one run would empty the lists."
|
|
exit 1
|
|
fi
|
|
log "$ICON_DOCKER docker reports $DOCKER_COUNT container(s); strike limit $LIMIT"
|
|
|
|
container_exists() { printf '%s\n' "$DOCKER_NAMES" | grep -qxF "$1"; }
|
|
|
|
# Names currently in the conf, per key, deduped for reporting.
|
|
declare -A SEEN_IN_KEYS=()
|
|
ALL_NAMES=()
|
|
for f in "${CONF_FILES[@]}"; do
|
|
[[ -f "$f" ]] || continue
|
|
for k in "${PRUNE_KEYS[@]}"; do
|
|
# Both array shapes: KEY=(a b c) on one line, or opened and closed across many.
|
|
vals=$(awk -v key="$k" '
|
|
$0 ~ "^[[:space:]]*"key"=\\(" { inside=1 }
|
|
inside { print }
|
|
inside && /\)/ && $0 !~ "^[[:space:]]*"key"=\\($" { inside=0 }
|
|
' "$f" | grep -oE '"[^"]+"' | tr -d '"')
|
|
for v in $vals; do
|
|
[[ -z "$v" ]] && continue
|
|
ALL_NAMES+=("$v")
|
|
SEEN_IN_KEYS["$v"]+="$(basename "$f"):$k "
|
|
done
|
|
done
|
|
done
|
|
|
|
if [[ ${#ALL_NAMES[@]} -eq 0 ]]; then
|
|
log "$ICON_DONE No container names in any prunable list — nothing to do"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "$MODE" == "status" ]]; then
|
|
echo "Strikes (limit $LIMIT):"
|
|
while IFS=: read -r name count; do
|
|
[[ -z "$name" ]] && continue
|
|
printf ' %-30s %s\n' "$name" "$count"
|
|
done < "$STRIKE_FILE"
|
|
exit 0
|
|
fi
|
|
|
|
# ── Strike, or clear ──────────────────────────────────────────────────────────────────────────
|
|
PRUNE=()
|
|
declare -A DONE=()
|
|
for name in "${ALL_NAMES[@]}"; do
|
|
[[ -n "${DONE[$name]:-}" ]] && continue
|
|
DONE["$name"]=1
|
|
|
|
if container_exists "$name"; then
|
|
prev=$(wd_state_get "$name" "$STRIKE_FILE")
|
|
if [[ -n "$prev" && "$prev" != "0" ]]; then
|
|
log " $name is back — clearing $prev strike(s)"
|
|
[[ "$DRY_RUN" == false ]] && wd_state_set "$name" 0 "$STRIKE_FILE"
|
|
fi
|
|
continue
|
|
fi
|
|
|
|
prev=$(wd_state_get "$name" "$STRIKE_FILE"); prev="${prev//[^0-9]/}"; prev="${prev:-0}"
|
|
# Pre-increment, never (( n++ )) — that returns 1 when n is 0 and would trip any errexit above.
|
|
next=$(( prev + 1 ))
|
|
if [[ "$next" -ge "$LIMIT" ]]; then
|
|
warn " $name missing from docker — strike $next/$LIMIT, PRUNING"
|
|
PRUNE+=("$name")
|
|
else
|
|
warn " $name missing from docker — strike $next/$LIMIT"
|
|
[[ "$DRY_RUN" == false ]] && wd_state_set "$name" "$next" "$STRIKE_FILE"
|
|
fi
|
|
done
|
|
|
|
if [[ ${#PRUNE[@]} -eq 0 ]]; then
|
|
log "$ICON_DONE Nothing at the strike limit"
|
|
exit 0
|
|
fi
|
|
|
|
# ── Remove, one name at a time, verifying after each file ─────────────────────────────────────
|
|
remove_from_conf() {
|
|
local file="$1" key="$2" name="$3"
|
|
local tmp; tmp=$(mktemp)
|
|
awk -v key="$key" -v name="\"$name\"" '
|
|
$0 ~ "^[[:space:]]*"key"=\\(" { inside=1 }
|
|
{
|
|
if (inside) {
|
|
# Single-line form: drop just the one quoted entry, keep the rest of the line.
|
|
if ($0 ~ /\)[[:space:]]*(#.*)?$/ && $0 ~ "^[[:space:]]*"key"=\\(") {
|
|
gsub(name"[[:space:]]*", "")
|
|
print; inside=0; next
|
|
}
|
|
# Multi-line form: a line that is only this entry disappears entirely.
|
|
line=$0; gsub(/^[[:space:]]+|[[:space:]]+$/, "", line)
|
|
if (line == name) next
|
|
if ($0 ~ /^[[:space:]]*\)/) inside=0
|
|
}
|
|
print
|
|
}
|
|
' "$file" > "$tmp" || { rm -f "$tmp"; return 1; }
|
|
|
|
# Accept only if the file still sources and the key still parses as an array.
|
|
if ! ( set +u; source "$tmp" >/dev/null 2>&1 ); then
|
|
rm -f "$tmp"; error " rewrite of $(basename "$file") would not source — kept the original"
|
|
return 1
|
|
fi
|
|
mv "$tmp" "$file"
|
|
return 0
|
|
}
|
|
|
|
for name in "${PRUNE[@]}"; do
|
|
for f in "${CONF_FILES[@]}"; do
|
|
[[ -f "$f" ]] || continue
|
|
for k in "${PRUNE_KEYS[@]}"; do
|
|
grep -qE "^[[:space:]]*$k=\(" "$f" || continue
|
|
grep -qF "\"$name\"" "$f" || continue
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn " DRY RUN — would remove $name from $k in $(basename "$f")"
|
|
continue
|
|
fi
|
|
cp "$f" "$f.bak-prune-$(date +%Y%m%d-%H%M%S)"
|
|
if remove_from_conf "$f" "$k" "$name"; then
|
|
log " removed $name from $k in $(basename "$f")"
|
|
fi
|
|
done
|
|
done
|
|
[[ "$DRY_RUN" == false ]] && wd_state_set "$name" 0 "$STRIKE_FILE"
|
|
done
|
|
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
notify "Pruned ${#PRUNE[@]} removed container(s) from conf on $(hostname) ($MY_ID): ${PRUNE[*]}" \
|
|
"Conf Container Prune" "normal"
|
|
fi
|
|
log "$ICON_DONE Pruned ${#PRUNE[@]} container(s): ${PRUNE[*]}"
|
|
exit 0
|