Add timed mutes so a temporary problem stops needing a permanent exemption

Every existing exemption lasts until someone remembers to undo it, and nobody
does — Healarr has sat in a pressure list since it was uninstalled and seven
ignore entries name containers that are gone. A mute states when it ends and
then ends, capped by WATCHDOG_MUTE_MAX_HOURS so temporary is enforced rather
than intended. Applied where IGNORE_MAP is built, so all five check sites
inherit it, and shown with its countdown because an invisible suppression is
the thing being fixed.
This commit is contained in:
Gmer4Lfe
2026-08-14 16:28:59 -04:00
parent 9f32644c32
commit 2cd1384786
5 changed files with 204 additions and 0 deletions
+95
View File
@@ -1788,6 +1788,101 @@ _lock_name() {
basename "${BASH_SOURCE[1]:-$0}" .sh
}
# ══════════════════════════════════════════════════════════════════════════════════════════════
# Timed mutes — a bounded, self-expiring exemption for one container
# ══════════════════════════════════════════════════════════════════════════════════════════════
# Every other exemption a watchdog offers is permanent: WATCHDOG_SCAN_IGNORE is a conf edit, an
# intentional stop lasts until it is cleared. So a container that needs quieting for an hour gets
# a workaround that outlives the reason for it, and nobody comes back — Healarr has sat in a
# pressure list since it was uninstalled, and seven names in the ignore list refer to containers
# that no longer exist. A mute is the missing shape: it states when it ends, and then it ends.
#
# Store format, one per line: container|until_epoch|reason
# Lives in STATE_DIR rather than conf because it is state, not configuration — and because a
# temporary decision written into conf is exactly the thing that stops being temporary.
#
# Fail-safe direction is deliberate and one-way: any problem reading the store yields NO mutes, so
# a corrupt or unreadable file makes the watchdogs behave normally rather than silently stop
# watching. Failing closed into silence is the one outcome a mute must never produce.
_wd_mute_file() {
echo "${WATCHDOG_MUTE_FILE:-${STATE_DIR:-/tmp}/watchdog_mutes.db}"
}
# Every container currently muted, one per line. Expired rows are simply not returned — expiry is
# a read-time comparison, so a mute ends on time whether or not anything prunes the file.
wd_mute_active() {
local f; f="$(_wd_mute_file)"
[[ -r "$f" ]] || return 0
local now; now=$(date +%s)
awk -F'|' -v now="$now" 'NF>=2 && $1!="" && $2+0>now {print $1}' "$f" 2>/dev/null
}
# Seconds remaining, or nothing when not muted. Used for reporting, never for control flow.
wd_mute_remaining() {
local f; f="$(_wd_mute_file)"
[[ -r "$f" ]] || return 0
local now; now=$(date +%s)
awk -F'|' -v now="$now" -v c="$1" 'NF>=2 && $1==c && $2+0>now {print $2-now; exit}' "$f" 2>/dev/null
}
wd_muted() {
[[ -n "$1" ]] || return 1
wd_mute_active | grep -qxF "$1"
}
# Drops expired rows. Called on every write so the file cannot grow without bound, and so the
# record on disk matches what is in force rather than accumulating history nobody reads.
wd_mute_prune() {
local f; f="$(_wd_mute_file)"
[[ -w "$f" ]] || return 0
local now tmp; now=$(date +%s); tmp="${f}.tmp.$$"
awk -F'|' -v now="$now" 'NF>=2 && $2+0>now' "$f" > "$tmp" 2>/dev/null && mv -f "$tmp" "$f"
rm -f "$tmp" 2>/dev/null
}
# wd_mute_add <container> <duration> [reason] duration: 45m | 2h | 1d
# Refuses anything past WATCHDOG_MUTE_MAX_HOURS. The cap is the whole point: without it this is
# just a slower way of writing an exemption that never expires.
wd_mute_add() {
local ctr="$1" dur="$2" reason="${3:-}"
[[ -n "$ctr" && -n "$dur" ]] || { echo "usage: wd_mute_add <container> <30m|2h|1d> [reason]" >&2; return 1; }
local n unit secs
n="${dur%[mhd]}"; unit="${dur##*[0-9]}"
[[ "$n" =~ ^[0-9]+$ && "$n" -gt 0 ]] || { echo "bad duration: $dur" >&2; return 1; }
case "$unit" in
m) secs=$(( n * 60 )) ;;
h) secs=$(( n * 3600 )) ;;
d) secs=$(( n * 86400 )) ;;
*) echo "bad duration unit: $dur (use m, h or d)" >&2; return 1 ;;
esac
local max=$(( ${WATCHDOG_MUTE_MAX_HOURS:-8} * 3600 ))
if (( secs > max )); then
echo "refused: ${dur} exceeds WATCHDOG_MUTE_MAX_HOURS=${WATCHDOG_MUTE_MAX_HOURS:-8}" >&2
return 1
fi
# A pipe would split the record and a newline would forge one, so neither is allowed through.
reason="${reason//|/ }"; reason="${reason//$'\n'/ }"
local f; f="$(_wd_mute_file)"
mkdir -p "$(dirname "$f")" 2>/dev/null
touch "$f" 2>/dev/null || { echo "cannot write $f" >&2; return 1; }
wd_mute_remove "$ctr" >/dev/null 2>&1 # re-muting replaces rather than stacks
printf '%s|%s|%s\n' "$ctr" "$(( $(date +%s) + secs ))" "$reason" >> "$f"
wd_mute_prune
}
wd_mute_remove() {
local ctr="$1" f; f="$(_wd_mute_file)"
[[ -n "$ctr" && -w "$f" ]] || return 0
local tmp="${f}.tmp.$$"
awk -F'|' -v c="$ctr" '$1!=c' "$f" > "$tmp" 2>/dev/null && mv -f "$tmp" "$f"
rm -f "$tmp" 2>/dev/null
}
# Internal — lock file path for this script
_lock_file() {
echo "$LOCK_DIR/${1:-$(_lock_name)}.lock"