Drop containers that no longer exist from the conf lists, after three days absent
A removed container leaves its name behind in every list that named it, and those names go on being acted upon — a watchdog waiting for a container docker has never heard of. Strikes rather than a single miss, because a rebuild makes a container vanish for a moment; and the lists naming containers on the REMOTE are excluded, since pruning those against local docker would empty them.
This commit is contained in:
Gmer4Lfe
Executable
+220
@@ -0,0 +1,220 @@
|
||||
#!/bin/bash
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# PURPOSE
|
||||
# Keep the conf's container lists honest. A container removed from docker leaves its name behind
|
||||
# in every list that referenced it, and those names go on being acted upon — a watchdog looking
|
||||
# for a required container that no longer exists, a fallback tier promising to start one, a
|
||||
# restart job reaching for a name docker has never heard of.
|
||||
#
|
||||
# OPERATIONAL MODEL
|
||||
# Strikes, never a single miss. A container is absent from `docker ps -a` for entirely ordinary
|
||||
# reasons: it is mid-rebuild, its image is being pulled, the operator is editing it. Five
|
||||
# containers on this host were briefly absent on 2026-08-23 while their templates were rewritten,
|
||||
# and a prune on first sight would have stripped all five from conf. A name must be missing on
|
||||
# CONF_PRUNE_STRIKE_LIMIT consecutive runs before anything is written.
|
||||
#
|
||||
# Seeing a container resets its strike to zero immediately, so a rebuild costs one strike at most.
|
||||
#
|
||||
# DESIGN PRINCIPLES
|
||||
# An explicit allow list of keys, never a pattern.
|
||||
# Not every list of container names describes THIS host. CRITICAL_CONTAINER_NAMES and
|
||||
# DELAYED_CONTAINERS are the containers stopped on the REMOTE before an rsync — pruning those
|
||||
# against local docker would empty them on a host that legitimately runs none of them, and
|
||||
# rsync would then stop nothing before copying a live database. They are excluded by name and
|
||||
# must stay excluded.
|
||||
#
|
||||
# A docker that cannot be read is not a docker with nothing in it.
|
||||
# If `docker ps -a` fails, times out, or returns nothing on a host that is meant to run
|
||||
# containers, EVERY name looks missing and one run would strike the entire configuration.
|
||||
# That case aborts before a single strike is recorded.
|
||||
#
|
||||
# The conf is backed up and re-read before the change is accepted.
|
||||
# An array can be written as KEY=(a b c) or spread over many lines, and this project has
|
||||
# already lost 68 vars to an editor that only understood one of those shapes. The rewrite is
|
||||
# verified by sourcing the result in a subshell and confirming the key still parses as an
|
||||
# array with exactly one fewer element.
|
||||
#
|
||||
# USAGE
|
||||
# conf_container_prune.sh strike, and prune anything at the limit
|
||||
# conf_container_prune.sh --dry-run report what would be struck and pruned, write nothing
|
||||
# conf_container_prune.sh --status show current strikes and stop
|
||||
#
|
||||
# DEPENDS ON
|
||||
# CONF_PRUNE_STRIKE_LIMIT consecutive misses before a name is removed (master.conf, default 3)
|
||||
# STATE_DIR/conf_prune_strikes.db the strike counts, one per container
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
DRY_RUN="${DRY_RUN:-false}"
|
||||
MODE="run"
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
--dry-run) DRY_RUN=true ;;
|
||||
--status) MODE="status" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
detect_hosts
|
||||
LIMIT="${CONF_PRUNE_STRIKE_LIMIT:-3}"
|
||||
STRIKE_FILE="${STATE_DIR}/conf_prune_strikes.db"
|
||||
[[ -f "$STRIKE_FILE" ]] || : > "$STRIKE_FILE"
|
||||
|
||||
# ── Which keys describe containers on THIS host ───────────────────────────────────────────────
|
||||
# Deliberately enumerated. CRITICAL_CONTAINER_NAMES and DELAYED_CONTAINERS are absent on purpose —
|
||||
# see the design note above; they name containers on the REMOTE.
|
||||
PRUNE_KEYS=(
|
||||
"FALLBACK_${MY_ID}_TIER1" "FALLBACK_${MY_ID}_TIER2"
|
||||
"FALLBACK_${MY_ID}_TIER3" "FALLBACK_${MY_ID}_TIER4"
|
||||
"${MY_ID}_WATCHDOG_REQUIRED_CONTAINERS"
|
||||
"${MY_ID}_DAILY_RESTART_CONTAINERS"
|
||||
"${MY_ID}_WEEKLY_RESTART_CONTAINERS"
|
||||
"${MY_ID}_DDNS_CONTAINERS"
|
||||
"${MY_ID}_NETWORK_CONNECT_CONTAINERS"
|
||||
"${MY_ID}_RW_PAUSE_CONTAINERS"
|
||||
"${MY_ID}_RW_STOP_CONTAINERS"
|
||||
"${MY_ID}_PARTNERSHIP_OWN_CONTAINERS"
|
||||
"${MY_ID}_PARTNERSHIP_REPLACE_CONTAINERS"
|
||||
"${MY_ID}_PARTNERSHIP_ARR_REPLACE_CONTAINERS"
|
||||
"RW_CRITICAL_CONTAINERS"
|
||||
)
|
||||
CONF_FILES=("$CONF_DIR/master.conf" "$CONF_DIR/$(echo "$MY_ID" | tr '[:upper:]' '[:lower:]').conf")
|
||||
|
||||
# ── What docker actually has ──────────────────────────────────────────────────────────────────
|
||||
DOCKER_NAMES=$(timeout "${DOCKER_TIMEOUT:-30}" docker ps -a --format '{{.Names}}' 2>/dev/null)
|
||||
DOCKER_RC=$?
|
||||
DOCKER_COUNT=$(printf '%s\n' "$DOCKER_NAMES" | grep -c .)
|
||||
|
||||
if [[ "$DOCKER_RC" -ne 0 ]] || [[ "$DOCKER_COUNT" -eq 0 ]]; then
|
||||
error "$ICON_DOCKER docker returned $DOCKER_COUNT container(s) (rc=$DOCKER_RC) — refusing to strike anything"
|
||||
error " Every configured name would look missing, and one run would empty the lists."
|
||||
exit 1
|
||||
fi
|
||||
log "$ICON_DOCKER docker reports $DOCKER_COUNT container(s); strike limit $LIMIT"
|
||||
|
||||
container_exists() { printf '%s\n' "$DOCKER_NAMES" | grep -qxF "$1"; }
|
||||
|
||||
# Names currently in the conf, per key, deduped for reporting.
|
||||
declare -A SEEN_IN_KEYS=()
|
||||
ALL_NAMES=()
|
||||
for f in "${CONF_FILES[@]}"; do
|
||||
[[ -f "$f" ]] || continue
|
||||
for k in "${PRUNE_KEYS[@]}"; do
|
||||
# Both array shapes: KEY=(a b c) on one line, or opened and closed across many.
|
||||
vals=$(awk -v key="$k" '
|
||||
$0 ~ "^[[:space:]]*"key"=\\(" { inside=1 }
|
||||
inside { print }
|
||||
inside && /\)/ && $0 !~ "^[[:space:]]*"key"=\\($" { inside=0 }
|
||||
' "$f" | grep -oE '"[^"]+"' | tr -d '"')
|
||||
for v in $vals; do
|
||||
[[ -z "$v" ]] && continue
|
||||
ALL_NAMES+=("$v")
|
||||
SEEN_IN_KEYS["$v"]+="$(basename "$f"):$k "
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
if [[ ${#ALL_NAMES[@]} -eq 0 ]]; then
|
||||
log "$ICON_DONE No container names in any prunable list — nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$MODE" == "status" ]]; then
|
||||
echo "Strikes (limit $LIMIT):"
|
||||
while IFS=: read -r name count; do
|
||||
[[ -z "$name" ]] && continue
|
||||
printf ' %-30s %s\n' "$name" "$count"
|
||||
done < "$STRIKE_FILE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Strike, or clear ──────────────────────────────────────────────────────────────────────────
|
||||
PRUNE=()
|
||||
declare -A DONE=()
|
||||
for name in "${ALL_NAMES[@]}"; do
|
||||
[[ -n "${DONE[$name]:-}" ]] && continue
|
||||
DONE["$name"]=1
|
||||
|
||||
if container_exists "$name"; then
|
||||
prev=$(wd_state_get "$name" "$STRIKE_FILE")
|
||||
if [[ -n "$prev" && "$prev" != "0" ]]; then
|
||||
log " $name is back — clearing $prev strike(s)"
|
||||
[[ "$DRY_RUN" == false ]] && wd_state_set "$name" 0 "$STRIKE_FILE"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
prev=$(wd_state_get "$name" "$STRIKE_FILE"); prev="${prev//[^0-9]/}"; prev="${prev:-0}"
|
||||
# Pre-increment, never (( n++ )) — that returns 1 when n is 0 and would trip any errexit above.
|
||||
next=$(( prev + 1 ))
|
||||
if [[ "$next" -ge "$LIMIT" ]]; then
|
||||
warn " $name missing from docker — strike $next/$LIMIT, PRUNING"
|
||||
PRUNE+=("$name")
|
||||
else
|
||||
warn " $name missing from docker — strike $next/$LIMIT"
|
||||
[[ "$DRY_RUN" == false ]] && wd_state_set "$name" "$next" "$STRIKE_FILE"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${#PRUNE[@]} -eq 0 ]]; then
|
||||
log "$ICON_DONE Nothing at the strike limit"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Remove, one name at a time, verifying after each file ─────────────────────────────────────
|
||||
remove_from_conf() {
|
||||
local file="$1" key="$2" name="$3"
|
||||
local tmp; tmp=$(mktemp)
|
||||
awk -v key="$key" -v name="\"$name\"" '
|
||||
$0 ~ "^[[:space:]]*"key"=\\(" { inside=1 }
|
||||
{
|
||||
if (inside) {
|
||||
# Single-line form: drop just the one quoted entry, keep the rest of the line.
|
||||
if ($0 ~ /\)[[:space:]]*(#.*)?$/ && $0 ~ "^[[:space:]]*"key"=\\(") {
|
||||
gsub(name"[[:space:]]*", "")
|
||||
print; inside=0; next
|
||||
}
|
||||
# Multi-line form: a line that is only this entry disappears entirely.
|
||||
line=$0; gsub(/^[[:space:]]+|[[:space:]]+$/, "", line)
|
||||
if (line == name) next
|
||||
if ($0 ~ /^[[:space:]]*\)/) inside=0
|
||||
}
|
||||
print
|
||||
}
|
||||
' "$file" > "$tmp" || { rm -f "$tmp"; return 1; }
|
||||
|
||||
# Accept only if the file still sources and the key still parses as an array.
|
||||
if ! ( set +u; source "$tmp" >/dev/null 2>&1 ); then
|
||||
rm -f "$tmp"; error " rewrite of $(basename "$file") would not source — kept the original"
|
||||
return 1
|
||||
fi
|
||||
mv "$tmp" "$file"
|
||||
return 0
|
||||
}
|
||||
|
||||
for name in "${PRUNE[@]}"; do
|
||||
for f in "${CONF_FILES[@]}"; do
|
||||
[[ -f "$f" ]] || continue
|
||||
for k in "${PRUNE_KEYS[@]}"; do
|
||||
grep -qE "^[[:space:]]*$k=\(" "$f" || continue
|
||||
grep -qF "\"$name\"" "$f" || continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn " DRY RUN — would remove $name from $k in $(basename "$f")"
|
||||
continue
|
||||
fi
|
||||
cp "$f" "$f.bak-prune-$(date +%Y%m%d-%H%M%S)"
|
||||
if remove_from_conf "$f" "$k" "$name"; then
|
||||
log " removed $name from $k in $(basename "$f")"
|
||||
fi
|
||||
done
|
||||
done
|
||||
[[ "$DRY_RUN" == false ]] && wd_state_set "$name" 0 "$STRIKE_FILE"
|
||||
done
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
notify "Pruned ${#PRUNE[@]} removed container(s) from conf on $(hostname) ($MY_ID): ${PRUNE[*]}" \
|
||||
"Conf Container Prune" "normal"
|
||||
fi
|
||||
log "$ICON_DONE Pruned ${#PRUNE[@]} container(s): ${PRUNE[*]}"
|
||||
exit 0
|
||||
Reference in New Issue
Block a user