Add timed mutes so a temporary problem stops needing a permanent exemption
Every existing exemption lasts until someone remembers to undo it, and nobody does — Healarr has sat in a pressure list since it was uninstalled and seven ignore entries name containers that are gone. A mute states when it ends and then ends, capped by WATCHDOG_MUTE_MAX_HOURS so temporary is enforced rather than intended. Applied where IGNORE_MAP is built, so all five check sites inherit it, and shown with its countdown because an invisible suppression is the thing being fixed.
This commit is contained in:
Gmer4Lfe
@@ -1600,9 +1600,17 @@
|
||||
SYS_WATCHDOG_STATE_FILE="$STATE_DIR/system_watchdog_state.db"
|
||||
DOCKER_WATCHDOG_FAILED_FILE="$STATE_DIR/docker_watchdog_failed.db"
|
||||
DOCKER_WATCHDOG_INTENTIONAL_FILE="$STATE_DIR/docker_intentional_stops.db"
|
||||
WATCHDOG_MUTE_FILE="$STATE_DIR/watchdog_mutes.db"
|
||||
SYS_WATCHDOG_REBOOT_LOG="$STATE_DIR/system_watchdog_reboots.db"
|
||||
SYS_WATCHDOG_OOM_FILE="$STATE_DIR/system_watchdog_oom.db"
|
||||
|
||||
# ━━━ Timed Mutes ━━━
|
||||
# A mute silences one container for a bounded time and then expires on its own. It exists because
|
||||
# every other exemption here is permanent — SCAN_IGNORE is a conf edit, an intentional stop lasts
|
||||
# until cleared — so a temporary problem gets a permanent workaround that nobody revisits.
|
||||
# Capped so "temporary" is enforced rather than intended: a mute longer than this is refused.
|
||||
WATCHDOG_MUTE_MAX_HOURS=8
|
||||
|
||||
# ━━━ Strike and Reboot Loop Settings ━━━
|
||||
# Strike system: a check must fail this many consecutive cycles before action is taken.
|
||||
# Single spikes (one bad reading) are ignored — sustained problems trigger reboot.
|
||||
|
||||
@@ -84,6 +84,31 @@ function vv_wd_installed_containers(): array {
|
||||
return array_values(array_filter(array_map('trim', $out), fn($n) => $n !== ''));
|
||||
}
|
||||
|
||||
// Active timed mutes, as [name => ['left' => seconds, 'reason' => text]]. Expiry is a read-time
|
||||
// comparison here for the same reason it is in wd_mute_active(): a mute ends when it says it does,
|
||||
// whether or not anything has pruned the file since.
|
||||
//
|
||||
// Surfaced because a suppression nobody can see is the problem this feature exists to fix. An
|
||||
// invisible exemption gets forgotten exactly like a permanent one — the only difference would be
|
||||
// that this one also lies about how long it lasts.
|
||||
function vv_wd_mutes(string $file = ''): array {
|
||||
$path = $file ?: (vv_conf_vars()['WATCHDOG_MUTE_FILE'] ?? STATE_DIR . '/watchdog_mutes.db');
|
||||
$raw = @file_get_contents($path);
|
||||
if ($raw === false) return [];
|
||||
$now = time();
|
||||
$out = [];
|
||||
foreach (explode("\n", $raw) as $line) {
|
||||
$line = trim($line);
|
||||
if ($line === '') continue;
|
||||
$p = explode('|', $line);
|
||||
if (count($p) < 2) continue;
|
||||
$until = (int)$p[1];
|
||||
if ($until <= $now || $p[0] === '') continue;
|
||||
$out[$p[0]] = ['left' => $until - $now, 'reason' => trim($p[2] ?? '')];
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
// How many minutes are supposed to pass between watchdog cycles.
|
||||
//
|
||||
// Parsed from varaverk.cron, which the scheduler regenerates at array start and is the only place
|
||||
@@ -294,6 +319,7 @@ function vv_wd_local_states(string $restartLogPath): array {
|
||||
// The heartbeat, not a strike — see VV_WD_SYS_BOOKKEEPING. stability_watchdog.sh writes it
|
||||
// last in the chain, so a fresh value means a whole cycle completed rather than started.
|
||||
'last_cycle' => (int)($sys['watchdog_cycle'] ?? 0),
|
||||
'mutes' => vv_wd_mutes(),
|
||||
'reboots' => vv_wd_parse_reboot_log($rebootRaw),
|
||||
'restarts' => vv_wd_parse_restart_log($restartRaw),
|
||||
'storage_wd' => vv_wd_parse_storage_state($storRaw) + [
|
||||
|
||||
@@ -514,6 +514,22 @@ function _dockerCard(node, cfg) {
|
||||
}
|
||||
}
|
||||
|
||||
// Shown with the time left, not just the name. A mute whose remaining time is invisible is
|
||||
// indistinguishable from the permanent exemptions it was built to replace — the countdown is
|
||||
// the entire difference, so it is the part that has to be on screen.
|
||||
const mutes = st.mutes || {};
|
||||
const muteNames = Object.keys(mutes);
|
||||
const muteHtml = muteNames.length === 0
|
||||
? '<div style="color:#333;font-size:11px;">none</div>'
|
||||
: `<div class="vv-wd-pill-row">${muteNames.map(n => {
|
||||
const m = mutes[n];
|
||||
const left = m.left >= 3600 ? Math.round(m.left / 3600) + 'h'
|
||||
: m.left >= 60 ? Math.round(m.left / 60) + 'm'
|
||||
: m.left + 's';
|
||||
return `<span class="vv-wd-pill warn" title="${vvEscAttr(m.reason || 'no reason given')}">`
|
||||
+ `${vvEscHtml(n)} · ${left}</span>`;
|
||||
}).join('')}</div>`;
|
||||
|
||||
let skipHtml = '';
|
||||
if (skiplist.length === 0) {
|
||||
skipHtml = '<div style="color:#333;font-size:11px;">empty</div>';
|
||||
@@ -548,6 +564,9 @@ function _dockerCard(node, cfg) {
|
||||
<div class="vv-wd-sec">Skip list</div>
|
||||
${skipHtml}
|
||||
<hr class="vv-wd-sep">
|
||||
<div class="vv-wd-sec">Muted</div>
|
||||
${muteHtml}
|
||||
<hr class="vv-wd-sep">
|
||||
<div class="vv-wd-sec">Restarts (24h)</div>
|
||||
${restartHtml}
|
||||
</div>`;
|
||||
|
||||
@@ -270,6 +270,18 @@
|
||||
# resumes on the next cycle. The container is not started — it remains stopped
|
||||
# until started manually.
|
||||
#
|
||||
# docker_watchdog.sh --mute ContainerName 2h "reason"
|
||||
# Silence every check for ContainerName until the time is up, then resume on its own.
|
||||
# Strikes, restarts, unhealthy and OOM reports and notifications are all suppressed —
|
||||
# the same suppression WATCHDOG_SCAN_IGNORE gives, with an end to it.
|
||||
# Duration is 30m, 2h or 1d, capped by WATCHDOG_MUTE_MAX_HOURS. Re-muting replaces.
|
||||
#
|
||||
# Use this, not --pause, when the container is meant to come back: --pause has no
|
||||
# expiry, which is how an exemption for one afternoon is still there a year later.
|
||||
#
|
||||
# docker_watchdog.sh --unmute ContainerName
|
||||
# End a mute early. Monitoring resumes on the next cycle.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -285,6 +297,10 @@ for (( _wdi=0; _wdi<${#PARSED_ARGS[@]}; _wdi++ )); do
|
||||
case "${PARSED_ARGS[$_wdi]}" in
|
||||
--pause) ((_wdi++)); WATCHDOG_PAUSE_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;;
|
||||
--resume) ((_wdi++)); WATCHDOG_RESUME_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;;
|
||||
--mute) ((_wdi++)); WATCHDOG_MUTE_CONTAINER="${PARSED_ARGS[$_wdi]:-}"
|
||||
((_wdi++)); WATCHDOG_MUTE_DURATION="${PARSED_ARGS[$_wdi]:-}"
|
||||
((_wdi++)); WATCHDOG_MUTE_REASON="${PARSED_ARGS[$_wdi]:-}" ;;
|
||||
--unmute) ((_wdi++)); WATCHDOG_UNMUTE_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;;
|
||||
esac
|
||||
done
|
||||
unset _wdi
|
||||
@@ -336,6 +352,13 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}"
|
||||
_intentional=$(cat "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
|
||||
echo "$ICON_SKIP Intentional: ${_intentional:-none}"
|
||||
_mutes=""
|
||||
while read -r _m; do
|
||||
[[ -n "$_m" ]] || continue
|
||||
_r=$(wd_mute_remaining "$_m")
|
||||
_mutes+="${_m}($(( ${_r:-0} / 60 ))m) "
|
||||
done < <(wd_mute_active)
|
||||
echo "$ICON_SKIP Muted: ${_mutes:-none}"
|
||||
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
|
||||
echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
|
||||
echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)"
|
||||
@@ -369,6 +392,31 @@ if [[ -n "$WATCHDOG_PAUSE_CONTAINER" || -n "$WATCHDOG_RESUME_CONTAINER" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Timed mutes — --mute / --unmute ──────────────────────────────────────────────────────────
|
||||
# Distinct from --pause, and the difference is the point. An intentional stop says "this is meant
|
||||
# to be down, leave it alone" and lasts until it is cleared. A mute says "leave it alone until
|
||||
# quarter past four", and then stops on its own — which is what a rebuild, a migration or a
|
||||
# vendor's broken update actually needs, and what nobody remembers to undo.
|
||||
if [[ -n "${WATCHDOG_MUTE_CONTAINER:-}" || -n "${WATCHDOG_UNMUTE_CONTAINER:-}" ]]; then
|
||||
if [[ -n "${WATCHDOG_MUTE_CONTAINER:-}" ]]; then
|
||||
if [[ -z "${WATCHDOG_MUTE_DURATION:-}" ]]; then
|
||||
error "--mute needs a duration: --mute <container> <30m|2h|1d> [reason]"
|
||||
exit 1
|
||||
fi
|
||||
if wd_mute_add "$WATCHDOG_MUTE_CONTAINER" "$WATCHDOG_MUTE_DURATION" "${WATCHDOG_MUTE_REASON:-}"; then
|
||||
_left=$(wd_mute_remaining "$WATCHDOG_MUTE_CONTAINER")
|
||||
success "$WATCHDOG_MUTE_CONTAINER muted for ${WATCHDOG_MUTE_DURATION} — expires $(date -d "@$(( $(date +%s) + ${_left:-0} ))" '+%H:%M' 2>/dev/null)"
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
if [[ -n "${WATCHDOG_UNMUTE_CONTAINER:-}" ]]; then
|
||||
wd_mute_remove "$WATCHDOG_UNMUTE_CONTAINER"
|
||||
success "$WATCHDOG_UNMUTE_CONTAINER unmuted — normal monitoring resumes next cycle"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
@@ -707,6 +755,14 @@ CYCLE_START=$(date +%s)
|
||||
for c in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do
|
||||
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
|
||||
done
|
||||
# A mute is a time-boxed ignore entry, so it is applied where the ignore map is built rather
|
||||
# than at each of the five places that consult it. Every existing check — strikes, restarts,
|
||||
# unhealthy, OOM, dependencies — inherits it without being touched, and nothing can be added
|
||||
# later that respects the ignore list but silently misses mutes.
|
||||
while read -r _muted; do
|
||||
[[ -n "$_muted" ]] && IGNORE_MAP["$_muted"]=1
|
||||
done < <(wd_mute_active)
|
||||
unset _muted
|
||||
|
||||
# ── Skip list and intentional stops visibility ───────────────────────────────────────────
|
||||
# Prune entries for containers that no longer exist at all (uninstalled/removed) from both
|
||||
|
||||
@@ -1788,6 +1788,101 @@ _lock_name() {
|
||||
basename "${BASH_SOURCE[1]:-$0}" .sh
|
||||
}
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# Timed mutes — a bounded, self-expiring exemption for one container
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# Every other exemption a watchdog offers is permanent: WATCHDOG_SCAN_IGNORE is a conf edit, an
|
||||
# intentional stop lasts until it is cleared. So a container that needs quieting for an hour gets
|
||||
# a workaround that outlives the reason for it, and nobody comes back — Healarr has sat in a
|
||||
# pressure list since it was uninstalled, and seven names in the ignore list refer to containers
|
||||
# that no longer exist. A mute is the missing shape: it states when it ends, and then it ends.
|
||||
#
|
||||
# Store format, one per line: container|until_epoch|reason
|
||||
# Lives in STATE_DIR rather than conf because it is state, not configuration — and because a
|
||||
# temporary decision written into conf is exactly the thing that stops being temporary.
|
||||
#
|
||||
# Fail-safe direction is deliberate and one-way: any problem reading the store yields NO mutes, so
|
||||
# a corrupt or unreadable file makes the watchdogs behave normally rather than silently stop
|
||||
# watching. Failing closed into silence is the one outcome a mute must never produce.
|
||||
|
||||
_wd_mute_file() {
|
||||
echo "${WATCHDOG_MUTE_FILE:-${STATE_DIR:-/tmp}/watchdog_mutes.db}"
|
||||
}
|
||||
|
||||
# Every container currently muted, one per line. Expired rows are simply not returned — expiry is
|
||||
# a read-time comparison, so a mute ends on time whether or not anything prunes the file.
|
||||
wd_mute_active() {
|
||||
local f; f="$(_wd_mute_file)"
|
||||
[[ -r "$f" ]] || return 0
|
||||
local now; now=$(date +%s)
|
||||
awk -F'|' -v now="$now" 'NF>=2 && $1!="" && $2+0>now {print $1}' "$f" 2>/dev/null
|
||||
}
|
||||
|
||||
# Seconds remaining, or nothing when not muted. Used for reporting, never for control flow.
|
||||
wd_mute_remaining() {
|
||||
local f; f="$(_wd_mute_file)"
|
||||
[[ -r "$f" ]] || return 0
|
||||
local now; now=$(date +%s)
|
||||
awk -F'|' -v now="$now" -v c="$1" 'NF>=2 && $1==c && $2+0>now {print $2-now; exit}' "$f" 2>/dev/null
|
||||
}
|
||||
|
||||
wd_muted() {
|
||||
[[ -n "$1" ]] || return 1
|
||||
wd_mute_active | grep -qxF "$1"
|
||||
}
|
||||
|
||||
# Drops expired rows. Called on every write so the file cannot grow without bound, and so the
|
||||
# record on disk matches what is in force rather than accumulating history nobody reads.
|
||||
wd_mute_prune() {
|
||||
local f; f="$(_wd_mute_file)"
|
||||
[[ -w "$f" ]] || return 0
|
||||
local now tmp; now=$(date +%s); tmp="${f}.tmp.$$"
|
||||
awk -F'|' -v now="$now" 'NF>=2 && $2+0>now' "$f" > "$tmp" 2>/dev/null && mv -f "$tmp" "$f"
|
||||
rm -f "$tmp" 2>/dev/null
|
||||
}
|
||||
|
||||
# wd_mute_add <container> <duration> [reason] duration: 45m | 2h | 1d
|
||||
# Refuses anything past WATCHDOG_MUTE_MAX_HOURS. The cap is the whole point: without it this is
|
||||
# just a slower way of writing an exemption that never expires.
|
||||
wd_mute_add() {
|
||||
local ctr="$1" dur="$2" reason="${3:-}"
|
||||
[[ -n "$ctr" && -n "$dur" ]] || { echo "usage: wd_mute_add <container> <30m|2h|1d> [reason]" >&2; return 1; }
|
||||
|
||||
local n unit secs
|
||||
n="${dur%[mhd]}"; unit="${dur##*[0-9]}"
|
||||
[[ "$n" =~ ^[0-9]+$ && "$n" -gt 0 ]] || { echo "bad duration: $dur" >&2; return 1; }
|
||||
case "$unit" in
|
||||
m) secs=$(( n * 60 )) ;;
|
||||
h) secs=$(( n * 3600 )) ;;
|
||||
d) secs=$(( n * 86400 )) ;;
|
||||
*) echo "bad duration unit: $dur (use m, h or d)" >&2; return 1 ;;
|
||||
esac
|
||||
|
||||
local max=$(( ${WATCHDOG_MUTE_MAX_HOURS:-8} * 3600 ))
|
||||
if (( secs > max )); then
|
||||
echo "refused: ${dur} exceeds WATCHDOG_MUTE_MAX_HOURS=${WATCHDOG_MUTE_MAX_HOURS:-8}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# A pipe would split the record and a newline would forge one, so neither is allowed through.
|
||||
reason="${reason//|/ }"; reason="${reason//$'\n'/ }"
|
||||
|
||||
local f; f="$(_wd_mute_file)"
|
||||
mkdir -p "$(dirname "$f")" 2>/dev/null
|
||||
touch "$f" 2>/dev/null || { echo "cannot write $f" >&2; return 1; }
|
||||
wd_mute_remove "$ctr" >/dev/null 2>&1 # re-muting replaces rather than stacks
|
||||
printf '%s|%s|%s\n' "$ctr" "$(( $(date +%s) + secs ))" "$reason" >> "$f"
|
||||
wd_mute_prune
|
||||
}
|
||||
|
||||
wd_mute_remove() {
|
||||
local ctr="$1" f; f="$(_wd_mute_file)"
|
||||
[[ -n "$ctr" && -w "$f" ]] || return 0
|
||||
local tmp="${f}.tmp.$$"
|
||||
awk -F'|' -v c="$ctr" '$1!=c' "$f" > "$tmp" 2>/dev/null && mv -f "$tmp" "$f"
|
||||
rm -f "$tmp" 2>/dev/null
|
||||
}
|
||||
|
||||
# Internal — lock file path for this script
|
||||
_lock_file() {
|
||||
echo "$LOCK_DIR/${1:-$(_lock_name)}.lock"
|
||||
|
||||
Reference in New Issue
Block a user