Gmer4Lfe
fbf2b92167
A removed container leaves its name behind in every list that named it, and those names go on being acted upon — a watchdog waiting for a container docker has never heard of. Strikes rather than a single miss, because a rebuild makes a container vanish for a moment; and the lists naming containers on the REMOTE are excluded, since pruning those against local docker would empty them.
221 lines
9.9 KiB
Bash
Executable File
221 lines
9.9 KiB
Bash
Executable File
#!/bin/bash
|
|
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
|
# PURPOSE
|
|
# Keep the conf's container lists honest. A container removed from docker leaves its name behind
|
|
# in every list that referenced it, and those names go on being acted upon — a watchdog looking
|
|
# for a required container that no longer exists, a fallback tier promising to start one, a
|
|
# restart job reaching for a name docker has never heard of.
|
|
#
|
|
# OPERATIONAL MODEL
|
|
# Strikes, never a single miss. A container is absent from `docker ps -a` for entirely ordinary
|
|
# reasons: it is mid-rebuild, its image is being pulled, the operator is editing it. Five
|
|
# containers on this host were briefly absent on 2026-08-23 while their templates were rewritten,
|
|
# and a prune on first sight would have stripped all five from conf. A name must be missing on
|
|
# CONF_PRUNE_STRIKE_LIMIT consecutive runs before anything is written.
|
|
#
|
|
# Seeing a container resets its strike to zero immediately, so a rebuild costs one strike at most.
|
|
#
|
|
# DESIGN PRINCIPLES
|
|
# An explicit allow list of keys, never a pattern.
|
|
# Not every list of container names describes THIS host. CRITICAL_CONTAINER_NAMES and
|
|
# DELAYED_CONTAINERS are the containers stopped on the REMOTE before an rsync — pruning those
|
|
# against local docker would empty them on a host that legitimately runs none of them, and
|
|
# rsync would then stop nothing before copying a live database. They are excluded by name and
|
|
# must stay excluded.
|
|
#
|
|
# A docker that cannot be read is not a docker with nothing in it.
|
|
# If `docker ps -a` fails, times out, or returns nothing on a host that is meant to run
|
|
# containers, EVERY name looks missing and one run would strike the entire configuration.
|
|
# That case aborts before a single strike is recorded.
|
|
#
|
|
# The conf is backed up and re-read before the change is accepted.
|
|
# An array can be written as KEY=(a b c) or spread over many lines, and this project has
|
|
# already lost 68 vars to an editor that only understood one of those shapes. The rewrite is
|
|
# verified by sourcing the result in a subshell and confirming the key still parses as an
|
|
# array with exactly one fewer element.
|
|
#
|
|
# USAGE
|
|
# conf_container_prune.sh strike, and prune anything at the limit
|
|
# conf_container_prune.sh --dry-run report what would be struck and pruned, write nothing
|
|
# conf_container_prune.sh --status show current strikes and stop
|
|
#
|
|
# DEPENDS ON
|
|
# CONF_PRUNE_STRIKE_LIMIT consecutive misses before a name is removed (master.conf, default 3)
|
|
# STATE_DIR/conf_prune_strikes.db the strike counts, one per container
|
|
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
DRY_RUN="${DRY_RUN:-false}"
|
|
MODE="run"
|
|
for a in "$@"; do
|
|
case "$a" in
|
|
--dry-run) DRY_RUN=true ;;
|
|
--status) MODE="status" ;;
|
|
esac
|
|
done
|
|
|
|
detect_hosts
|
|
LIMIT="${CONF_PRUNE_STRIKE_LIMIT:-3}"
|
|
STRIKE_FILE="${STATE_DIR}/conf_prune_strikes.db"
|
|
[[ -f "$STRIKE_FILE" ]] || : > "$STRIKE_FILE"
|
|
|
|
# ── Which keys describe containers on THIS host ───────────────────────────────────────────────
|
|
# Deliberately enumerated. CRITICAL_CONTAINER_NAMES and DELAYED_CONTAINERS are absent on purpose —
|
|
# see the design note above; they name containers on the REMOTE.
|
|
PRUNE_KEYS=(
|
|
"FALLBACK_${MY_ID}_TIER1" "FALLBACK_${MY_ID}_TIER2"
|
|
"FALLBACK_${MY_ID}_TIER3" "FALLBACK_${MY_ID}_TIER4"
|
|
"${MY_ID}_WATCHDOG_REQUIRED_CONTAINERS"
|
|
"${MY_ID}_DAILY_RESTART_CONTAINERS"
|
|
"${MY_ID}_WEEKLY_RESTART_CONTAINERS"
|
|
"${MY_ID}_DDNS_CONTAINERS"
|
|
"${MY_ID}_NETWORK_CONNECT_CONTAINERS"
|
|
"${MY_ID}_RW_PAUSE_CONTAINERS"
|
|
"${MY_ID}_RW_STOP_CONTAINERS"
|
|
"${MY_ID}_PARTNERSHIP_OWN_CONTAINERS"
|
|
"${MY_ID}_PARTNERSHIP_REPLACE_CONTAINERS"
|
|
"${MY_ID}_PARTNERSHIP_ARR_REPLACE_CONTAINERS"
|
|
"RW_CRITICAL_CONTAINERS"
|
|
)
|
|
CONF_FILES=("$CONF_DIR/master.conf" "$CONF_DIR/$(echo "$MY_ID" | tr '[:upper:]' '[:lower:]').conf")
|
|
|
|
# ── What docker actually has ──────────────────────────────────────────────────────────────────
|
|
DOCKER_NAMES=$(timeout "${DOCKER_TIMEOUT:-30}" docker ps -a --format '{{.Names}}' 2>/dev/null)
|
|
DOCKER_RC=$?
|
|
DOCKER_COUNT=$(printf '%s\n' "$DOCKER_NAMES" | grep -c .)
|
|
|
|
if [[ "$DOCKER_RC" -ne 0 ]] || [[ "$DOCKER_COUNT" -eq 0 ]]; then
|
|
error "$ICON_DOCKER docker returned $DOCKER_COUNT container(s) (rc=$DOCKER_RC) — refusing to strike anything"
|
|
error " Every configured name would look missing, and one run would empty the lists."
|
|
exit 1
|
|
fi
|
|
log "$ICON_DOCKER docker reports $DOCKER_COUNT container(s); strike limit $LIMIT"
|
|
|
|
container_exists() { printf '%s\n' "$DOCKER_NAMES" | grep -qxF "$1"; }
|
|
|
|
# Names currently in the conf, per key, deduped for reporting.
|
|
declare -A SEEN_IN_KEYS=()
|
|
ALL_NAMES=()
|
|
for f in "${CONF_FILES[@]}"; do
|
|
[[ -f "$f" ]] || continue
|
|
for k in "${PRUNE_KEYS[@]}"; do
|
|
# Both array shapes: KEY=(a b c) on one line, or opened and closed across many.
|
|
vals=$(awk -v key="$k" '
|
|
$0 ~ "^[[:space:]]*"key"=\\(" { inside=1 }
|
|
inside { print }
|
|
inside && /\)/ && $0 !~ "^[[:space:]]*"key"=\\($" { inside=0 }
|
|
' "$f" | grep -oE '"[^"]+"' | tr -d '"')
|
|
for v in $vals; do
|
|
[[ -z "$v" ]] && continue
|
|
ALL_NAMES+=("$v")
|
|
SEEN_IN_KEYS["$v"]+="$(basename "$f"):$k "
|
|
done
|
|
done
|
|
done
|
|
|
|
if [[ ${#ALL_NAMES[@]} -eq 0 ]]; then
|
|
log "$ICON_DONE No container names in any prunable list — nothing to do"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "$MODE" == "status" ]]; then
|
|
echo "Strikes (limit $LIMIT):"
|
|
while IFS=: read -r name count; do
|
|
[[ -z "$name" ]] && continue
|
|
printf ' %-30s %s\n' "$name" "$count"
|
|
done < "$STRIKE_FILE"
|
|
exit 0
|
|
fi
|
|
|
|
# ── Strike, or clear ──────────────────────────────────────────────────────────────────────────
|
|
PRUNE=()
|
|
declare -A DONE=()
|
|
for name in "${ALL_NAMES[@]}"; do
|
|
[[ -n "${DONE[$name]:-}" ]] && continue
|
|
DONE["$name"]=1
|
|
|
|
if container_exists "$name"; then
|
|
prev=$(wd_state_get "$name" "$STRIKE_FILE")
|
|
if [[ -n "$prev" && "$prev" != "0" ]]; then
|
|
log " $name is back — clearing $prev strike(s)"
|
|
[[ "$DRY_RUN" == false ]] && wd_state_set "$name" 0 "$STRIKE_FILE"
|
|
fi
|
|
continue
|
|
fi
|
|
|
|
prev=$(wd_state_get "$name" "$STRIKE_FILE"); prev="${prev//[^0-9]/}"; prev="${prev:-0}"
|
|
# Pre-increment, never (( n++ )) — that returns 1 when n is 0 and would trip any errexit above.
|
|
next=$(( prev + 1 ))
|
|
if [[ "$next" -ge "$LIMIT" ]]; then
|
|
warn " $name missing from docker — strike $next/$LIMIT, PRUNING"
|
|
PRUNE+=("$name")
|
|
else
|
|
warn " $name missing from docker — strike $next/$LIMIT"
|
|
[[ "$DRY_RUN" == false ]] && wd_state_set "$name" "$next" "$STRIKE_FILE"
|
|
fi
|
|
done
|
|
|
|
if [[ ${#PRUNE[@]} -eq 0 ]]; then
|
|
log "$ICON_DONE Nothing at the strike limit"
|
|
exit 0
|
|
fi
|
|
|
|
# ── Remove, one name at a time, verifying after each file ─────────────────────────────────────
|
|
remove_from_conf() {
|
|
local file="$1" key="$2" name="$3"
|
|
local tmp; tmp=$(mktemp)
|
|
awk -v key="$key" -v name="\"$name\"" '
|
|
$0 ~ "^[[:space:]]*"key"=\\(" { inside=1 }
|
|
{
|
|
if (inside) {
|
|
# Single-line form: drop just the one quoted entry, keep the rest of the line.
|
|
if ($0 ~ /\)[[:space:]]*(#.*)?$/ && $0 ~ "^[[:space:]]*"key"=\\(") {
|
|
gsub(name"[[:space:]]*", "")
|
|
print; inside=0; next
|
|
}
|
|
# Multi-line form: a line that is only this entry disappears entirely.
|
|
line=$0; gsub(/^[[:space:]]+|[[:space:]]+$/, "", line)
|
|
if (line == name) next
|
|
if ($0 ~ /^[[:space:]]*\)/) inside=0
|
|
}
|
|
print
|
|
}
|
|
' "$file" > "$tmp" || { rm -f "$tmp"; return 1; }
|
|
|
|
# Accept only if the file still sources and the key still parses as an array.
|
|
if ! ( set +u; source "$tmp" >/dev/null 2>&1 ); then
|
|
rm -f "$tmp"; error " rewrite of $(basename "$file") would not source — kept the original"
|
|
return 1
|
|
fi
|
|
mv "$tmp" "$file"
|
|
return 0
|
|
}
|
|
|
|
for name in "${PRUNE[@]}"; do
|
|
for f in "${CONF_FILES[@]}"; do
|
|
[[ -f "$f" ]] || continue
|
|
for k in "${PRUNE_KEYS[@]}"; do
|
|
grep -qE "^[[:space:]]*$k=\(" "$f" || continue
|
|
grep -qF "\"$name\"" "$f" || continue
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn " DRY RUN — would remove $name from $k in $(basename "$f")"
|
|
continue
|
|
fi
|
|
cp "$f" "$f.bak-prune-$(date +%Y%m%d-%H%M%S)"
|
|
if remove_from_conf "$f" "$k" "$name"; then
|
|
log " removed $name from $k in $(basename "$f")"
|
|
fi
|
|
done
|
|
done
|
|
[[ "$DRY_RUN" == false ]] && wd_state_set "$name" 0 "$STRIKE_FILE"
|
|
done
|
|
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
notify "Pruned ${#PRUNE[@]} removed container(s) from conf on $(hostname) ($MY_ID): ${PRUNE[*]}" \
|
|
"Conf Container Prune" "normal"
|
|
fi
|
|
log "$ICON_DONE Pruned ${#PRUNE[@]} container(s): ${PRUNE[*]}"
|
|
exit 0
|