Add timed mutes so a temporary problem stops needing a permanent exemption
Every existing exemption lasts until someone remembers to undo it, and nobody does — Healarr has sat in a pressure list since it was uninstalled and seven ignore entries name containers that are gone. A mute states when it ends and then ends, capped by WATCHDOG_MUTE_MAX_HOURS so temporary is enforced rather than intended. Applied where IGNORE_MAP is built, so all five check sites inherit it, and shown with its countdown because an invisible suppression is the thing being fixed.
This commit is contained in:
@@ -1788,6 +1788,101 @@ _lock_name() {
|
||||
basename "${BASH_SOURCE[1]:-$0}" .sh
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# Timed mutes — a bounded, self-expiring exemption for one container
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# Every other exemption a watchdog offers is permanent: WATCHDOG_SCAN_IGNORE is a conf edit, an
|
||||
# intentional stop lasts until it is cleared. So a container that needs quieting for an hour gets
|
||||
# a workaround that outlives the reason for it, and nobody comes back — Healarr has sat in a
|
||||
# pressure list since it was uninstalled, and seven names in the ignore list refer to containers
|
||||
# that no longer exist. A mute is the missing shape: it states when it ends, and then it ends.
|
||||
#
|
||||
# Store format, one per line: container|until_epoch|reason
|
||||
# Lives in STATE_DIR rather than conf because it is state, not configuration — and because a
|
||||
# temporary decision written into conf is exactly the thing that stops being temporary.
|
||||
#
|
||||
# Fail-safe direction is deliberate and one-way: any problem reading the store yields NO mutes, so
|
||||
# a corrupt or unreadable file makes the watchdogs behave normally rather than silently stop
|
||||
# watching. Failing closed into silence is the one outcome a mute must never produce.
|
||||
|
||||
_wd_mute_file() {
|
||||
echo "${WATCHDOG_MUTE_FILE:-${STATE_DIR:-/tmp}/watchdog_mutes.db}"
|
||||
}
|
||||
|
||||
# Every container currently muted, one per line. Expired rows are simply not returned — expiry is
|
||||
# a read-time comparison, so a mute ends on time whether or not anything prunes the file.
|
||||
wd_mute_active() {
|
||||
local f; f="$(_wd_mute_file)"
|
||||
[[ -r "$f" ]] || return 0
|
||||
local now; now=$(date +%s)
|
||||
awk -F'|' -v now="$now" 'NF>=2 && $1!="" && $2+0>now {print $1}' "$f" 2>/dev/null
|
||||
}
|
||||
|
||||
# Seconds remaining, or nothing when not muted. Used for reporting, never for control flow.
|
||||
wd_mute_remaining() {
|
||||
local f; f="$(_wd_mute_file)"
|
||||
[[ -r "$f" ]] || return 0
|
||||
local now; now=$(date +%s)
|
||||
awk -F'|' -v now="$now" -v c="$1" 'NF>=2 && $1==c && $2+0>now {print $2-now; exit}' "$f" 2>/dev/null
|
||||
}
|
||||
|
||||
wd_muted() {
|
||||
[[ -n "$1" ]] || return 1
|
||||
wd_mute_active | grep -qxF "$1"
|
||||
}
|
||||
|
||||
# Drops expired rows. Called on every write so the file cannot grow without bound, and so the
|
||||
# record on disk matches what is in force rather than accumulating history nobody reads.
|
||||
wd_mute_prune() {
|
||||
local f; f="$(_wd_mute_file)"
|
||||
[[ -w "$f" ]] || return 0
|
||||
local now tmp; now=$(date +%s); tmp="${f}.tmp.$$"
|
||||
awk -F'|' -v now="$now" 'NF>=2 && $2+0>now' "$f" > "$tmp" 2>/dev/null && mv -f "$tmp" "$f"
|
||||
rm -f "$tmp" 2>/dev/null
|
||||
}
|
||||
|
||||
# wd_mute_add <container> <duration> [reason] duration: 45m | 2h | 1d
|
||||
# Refuses anything past WATCHDOG_MUTE_MAX_HOURS. The cap is the whole point: without it this is
|
||||
# just a slower way of writing an exemption that never expires.
|
||||
wd_mute_add() {
|
||||
local ctr="$1" dur="$2" reason="${3:-}"
|
||||
[[ -n "$ctr" && -n "$dur" ]] || { echo "usage: wd_mute_add <container> <30m|2h|1d> [reason]" >&2; return 1; }
|
||||
|
||||
local n unit secs
|
||||
n="${dur%[mhd]}"; unit="${dur##*[0-9]}"
|
||||
[[ "$n" =~ ^[0-9]+$ && "$n" -gt 0 ]] || { echo "bad duration: $dur" >&2; return 1; }
|
||||
case "$unit" in
|
||||
m) secs=$(( n * 60 )) ;;
|
||||
h) secs=$(( n * 3600 )) ;;
|
||||
d) secs=$(( n * 86400 )) ;;
|
||||
*) echo "bad duration unit: $dur (use m, h or d)" >&2; return 1 ;;
|
||||
esac
|
||||
|
||||
local max=$(( ${WATCHDOG_MUTE_MAX_HOURS:-8} * 3600 ))
|
||||
if (( secs > max )); then
|
||||
echo "refused: ${dur} exceeds WATCHDOG_MUTE_MAX_HOURS=${WATCHDOG_MUTE_MAX_HOURS:-8}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# A pipe would split the record and a newline would forge one, so neither is allowed through.
|
||||
reason="${reason//|/ }"; reason="${reason//$'\n'/ }"
|
||||
|
||||
local f; f="$(_wd_mute_file)"
|
||||
mkdir -p "$(dirname "$f")" 2>/dev/null
|
||||
touch "$f" 2>/dev/null || { echo "cannot write $f" >&2; return 1; }
|
||||
wd_mute_remove "$ctr" >/dev/null 2>&1 # re-muting replaces rather than stacks
|
||||
printf '%s|%s|%s\n' "$ctr" "$(( $(date +%s) + secs ))" "$reason" >> "$f"
|
||||
wd_mute_prune
|
||||
}
|
||||
|
||||
wd_mute_remove() {
|
||||
local ctr="$1" f; f="$(_wd_mute_file)"
|
||||
[[ -n "$ctr" && -w "$f" ]] || return 0
|
||||
local tmp="${f}.tmp.$$"
|
||||
awk -F'|' -v c="$ctr" '$1!=c' "$f" > "$tmp" 2>/dev/null && mv -f "$tmp" "$f"
|
||||
rm -f "$tmp" 2>/dev/null
|
||||
}
|
||||
|
||||
# Internal — lock file path for this script
|
||||
_lock_file() {
|
||||
echo "$LOCK_DIR/${1:-$(_lock_name)}.lock"
|
||||
|
||||
Reference in New Issue
Block a user