diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index a84946d..258a23b 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -1600,9 +1600,17 @@ SYS_WATCHDOG_STATE_FILE="$STATE_DIR/system_watchdog_state.db" DOCKER_WATCHDOG_FAILED_FILE="$STATE_DIR/docker_watchdog_failed.db" DOCKER_WATCHDOG_INTENTIONAL_FILE="$STATE_DIR/docker_intentional_stops.db" + WATCHDOG_MUTE_FILE="$STATE_DIR/watchdog_mutes.db" SYS_WATCHDOG_REBOOT_LOG="$STATE_DIR/system_watchdog_reboots.db" SYS_WATCHDOG_OOM_FILE="$STATE_DIR/system_watchdog_oom.db" +# ━━━ Timed Mutes ━━━ +# A mute silences one container for a bounded time and then expires on its own. It exists because +# every other exemption here is permanent — SCAN_IGNORE is a conf edit, an intentional stop lasts +# until cleared — so a temporary problem gets a permanent workaround that nobody revisits. +# Capped so "temporary" is enforced rather than intended: a mute longer than this is refused. + WATCHDOG_MUTE_MAX_HOURS=8 + # ━━━ Strike and Reboot Loop Settings ━━━ # Strike system: a check must fail this many consecutive cycles before action is taken. # Single spikes (one bad reading) are ignored — sustained problems trigger reboot. diff --git a/Plugin/unraid/include/watchdog.php b/Plugin/unraid/include/watchdog.php index da3e710..bbafb05 100644 --- a/Plugin/unraid/include/watchdog.php +++ b/Plugin/unraid/include/watchdog.php @@ -84,6 +84,31 @@ function vv_wd_installed_containers(): array { return array_values(array_filter(array_map('trim', $out), fn($n) => $n !== '')); } +// Active timed mutes, as [name => ['left' => seconds, 'reason' => text]]. Expiry is a read-time +// comparison here for the same reason it is in wd_mute_active(): a mute ends when it says it does, +// whether or not anything has pruned the file since. +// +// Surfaced because a suppression nobody can see is the problem this feature exists to fix. An +// invisible exemption gets forgotten exactly like a permanent one — the only difference would be +// that this one also lies about how long it lasts. +function vv_wd_mutes(string $file = ''): array { + $path = $file ?: (vv_conf_vars()['WATCHDOG_MUTE_FILE'] ?? STATE_DIR . '/watchdog_mutes.db'); + $raw = @file_get_contents($path); + if ($raw === false) return []; + $now = time(); + $out = []; + foreach (explode("\n", $raw) as $line) { + $line = trim($line); + if ($line === '') continue; + $p = explode('|', $line); + if (count($p) < 2) continue; + $until = (int)$p[1]; + if ($until <= $now || $p[0] === '') continue; + $out[$p[0]] = ['left' => $until - $now, 'reason' => trim($p[2] ?? '')]; + } + return $out; +} + // How many minutes are supposed to pass between watchdog cycles. // // Parsed from varaverk.cron, which the scheduler regenerates at array start and is the only place @@ -294,6 +319,7 @@ function vv_wd_local_states(string $restartLogPath): array { // The heartbeat, not a strike — see VV_WD_SYS_BOOKKEEPING. stability_watchdog.sh writes it // last in the chain, so a fresh value means a whole cycle completed rather than started. 'last_cycle' => (int)($sys['watchdog_cycle'] ?? 0), + 'mutes' => vv_wd_mutes(), 'reboots' => vv_wd_parse_reboot_log($rebootRaw), 'restarts' => vv_wd_parse_restart_log($restartRaw), 'storage_wd' => vv_wd_parse_storage_state($storRaw) + [ diff --git a/Plugin/unraid/pages/watchdog.php b/Plugin/unraid/pages/watchdog.php index 6711a9b..2d5552b 100644 --- a/Plugin/unraid/pages/watchdog.php +++ b/Plugin/unraid/pages/watchdog.php @@ -514,6 +514,22 @@ function _dockerCard(node, cfg) { } } + // Shown with the time left, not just the name. A mute whose remaining time is invisible is + // indistinguishable from the permanent exemptions it was built to replace — the countdown is + // the entire difference, so it is the part that has to be on screen. + const mutes = st.mutes || {}; + const muteNames = Object.keys(mutes); + const muteHtml = muteNames.length === 0 + ? '
none
' + : `
${muteNames.map(n => { + const m = mutes[n]; + const left = m.left >= 3600 ? Math.round(m.left / 3600) + 'h' + : m.left >= 60 ? Math.round(m.left / 60) + 'm' + : m.left + 's'; + return `` + + `${vvEscHtml(n)} · ${left}`; + }).join('')}
`; + let skipHtml = ''; if (skiplist.length === 0) { skipHtml = '
empty
'; @@ -548,6 +564,9 @@ function _dockerCard(node, cfg) {
Skip list
${skipHtml}
+
Muted
+ ${muteHtml} +
Restarts (24h)
${restartHtml} `; diff --git a/Watchdogs/docker_watchdog.sh b/Watchdogs/docker_watchdog.sh index 6f25d53..1edd401 100755 --- a/Watchdogs/docker_watchdog.sh +++ b/Watchdogs/docker_watchdog.sh @@ -270,6 +270,18 @@ # resumes on the next cycle. The container is not started — it remains stopped # until started manually. # +# docker_watchdog.sh --mute ContainerName 2h "reason" +# Silence every check for ContainerName until the time is up, then resume on its own. +# Strikes, restarts, unhealthy and OOM reports and notifications are all suppressed — +# the same suppression WATCHDOG_SCAN_IGNORE gives, with an end to it. +# Duration is 30m, 2h or 1d, capped by WATCHDOG_MUTE_MAX_HOURS. Re-muting replaces. +# +# Use this, not --pause, when the container is meant to come back: --pause has no +# expiry, which is how an exemption for one afternoon is still there a year later. +# +# docker_watchdog.sh --unmute ContainerName +# End a mute early. Monitoring resumes on the next cycle. +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -285,6 +297,10 @@ for (( _wdi=0; _wdi<${#PARSED_ARGS[@]}; _wdi++ )); do case "${PARSED_ARGS[$_wdi]}" in --pause) ((_wdi++)); WATCHDOG_PAUSE_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;; --resume) ((_wdi++)); WATCHDOG_RESUME_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;; + --mute) ((_wdi++)); WATCHDOG_MUTE_CONTAINER="${PARSED_ARGS[$_wdi]:-}" + ((_wdi++)); WATCHDOG_MUTE_DURATION="${PARSED_ARGS[$_wdi]:-}" + ((_wdi++)); WATCHDOG_MUTE_REASON="${PARSED_ARGS[$_wdi]:-}" ;; + --unmute) ((_wdi++)); WATCHDOG_UNMUTE_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;; esac done unset _wdi @@ -336,6 +352,13 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}" _intentional=$(cat "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null | tr '\n' ' ' | xargs) echo "$ICON_SKIP Intentional: ${_intentional:-none}" + _mutes="" + while read -r _m; do + [[ -n "$_m" ]] || continue + _r=$(wd_mute_remaining "$_m") + _mutes+="${_m}($(( ${_r:-0} / 60 ))m) " + done < <(wd_mute_active) + echo "$ICON_SKIP Muted: ${_mutes:-none}" echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL" echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}" echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)" @@ -369,6 +392,31 @@ if [[ -n "$WATCHDOG_PAUSE_CONTAINER" || -n "$WATCHDOG_RESUME_CONTAINER" ]]; then exit 0 fi +# ── Timed mutes — --mute / --unmute ────────────────────────────────────────────────────────── +# Distinct from --pause, and the difference is the point. An intentional stop says "this is meant +# to be down, leave it alone" and lasts until it is cleared. A mute says "leave it alone until +# quarter past four", and then stops on its own — which is what a rebuild, a migration or a +# vendor's broken update actually needs, and what nobody remembers to undo. +if [[ -n "${WATCHDOG_MUTE_CONTAINER:-}" || -n "${WATCHDOG_UNMUTE_CONTAINER:-}" ]]; then + if [[ -n "${WATCHDOG_MUTE_CONTAINER:-}" ]]; then + if [[ -z "${WATCHDOG_MUTE_DURATION:-}" ]]; then + error "--mute needs a duration: --mute <30m|2h|1d> [reason]" + exit 1 + fi + if wd_mute_add "$WATCHDOG_MUTE_CONTAINER" "$WATCHDOG_MUTE_DURATION" "${WATCHDOG_MUTE_REASON:-}"; then + _left=$(wd_mute_remaining "$WATCHDOG_MUTE_CONTAINER") + success "$WATCHDOG_MUTE_CONTAINER muted for ${WATCHDOG_MUTE_DURATION} — expires $(date -d "@$(( $(date +%s) + ${_left:-0} ))" '+%H:%M' 2>/dev/null)" + else + exit 1 + fi + fi + if [[ -n "${WATCHDOG_UNMUTE_CONTAINER:-}" ]]; then + wd_mute_remove "$WATCHDOG_UNMUTE_CONTAINER" + success "$WATCHDOG_UNMUTE_CONTAINER unmuted — normal monitoring resumes next cycle" + fi + exit 0 +fi + # ============================================================================================== # ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -707,6 +755,14 @@ CYCLE_START=$(date +%s) for c in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do [[ -n "$c" ]] && IGNORE_MAP["$c"]=1 done + # A mute is a time-boxed ignore entry, so it is applied where the ignore map is built rather + # than at each of the five places that consult it. Every existing check — strikes, restarts, + # unhealthy, OOM, dependencies — inherits it without being touched, and nothing can be added + # later that respects the ignore list but silently misses mutes. + while read -r _muted; do + [[ -n "$_muted" ]] && IGNORE_MAP["$_muted"]=1 + done < <(wd_mute_active) + unset _muted # ── Skip list and intentional stops visibility ─────────────────────────────────────────── # Prune entries for containers that no longer exist at all (uninstalled/removed) from both diff --git a/common.sh b/common.sh index 6e2ca2f..9dca2b2 100755 --- a/common.sh +++ b/common.sh @@ -1788,6 +1788,101 @@ _lock_name() { basename "${BASH_SOURCE[1]:-$0}" .sh } +# ══════════════════════════════════════════════════════════════════════════════════════════════ +# Timed mutes — a bounded, self-expiring exemption for one container +# ══════════════════════════════════════════════════════════════════════════════════════════════ +# Every other exemption a watchdog offers is permanent: WATCHDOG_SCAN_IGNORE is a conf edit, an +# intentional stop lasts until it is cleared. So a container that needs quieting for an hour gets +# a workaround that outlives the reason for it, and nobody comes back — Healarr has sat in a +# pressure list since it was uninstalled, and seven names in the ignore list refer to containers +# that no longer exist. A mute is the missing shape: it states when it ends, and then it ends. +# +# Store format, one per line: container|until_epoch|reason +# Lives in STATE_DIR rather than conf because it is state, not configuration — and because a +# temporary decision written into conf is exactly the thing that stops being temporary. +# +# Fail-safe direction is deliberate and one-way: any problem reading the store yields NO mutes, so +# a corrupt or unreadable file makes the watchdogs behave normally rather than silently stop +# watching. Failing closed into silence is the one outcome a mute must never produce. + +_wd_mute_file() { + echo "${WATCHDOG_MUTE_FILE:-${STATE_DIR:-/tmp}/watchdog_mutes.db}" +} + +# Every container currently muted, one per line. Expired rows are simply not returned — expiry is +# a read-time comparison, so a mute ends on time whether or not anything prunes the file. +wd_mute_active() { + local f; f="$(_wd_mute_file)" + [[ -r "$f" ]] || return 0 + local now; now=$(date +%s) + awk -F'|' -v now="$now" 'NF>=2 && $1!="" && $2+0>now {print $1}' "$f" 2>/dev/null +} + +# Seconds remaining, or nothing when not muted. Used for reporting, never for control flow. +wd_mute_remaining() { + local f; f="$(_wd_mute_file)" + [[ -r "$f" ]] || return 0 + local now; now=$(date +%s) + awk -F'|' -v now="$now" -v c="$1" 'NF>=2 && $1==c && $2+0>now {print $2-now; exit}' "$f" 2>/dev/null +} + +wd_muted() { + [[ -n "$1" ]] || return 1 + wd_mute_active | grep -qxF "$1" +} + +# Drops expired rows. Called on every write so the file cannot grow without bound, and so the +# record on disk matches what is in force rather than accumulating history nobody reads. +wd_mute_prune() { + local f; f="$(_wd_mute_file)" + [[ -w "$f" ]] || return 0 + local now tmp; now=$(date +%s); tmp="${f}.tmp.$$" + awk -F'|' -v now="$now" 'NF>=2 && $2+0>now' "$f" > "$tmp" 2>/dev/null && mv -f "$tmp" "$f" + rm -f "$tmp" 2>/dev/null +} + +# wd_mute_add [reason] duration: 45m | 2h | 1d +# Refuses anything past WATCHDOG_MUTE_MAX_HOURS. The cap is the whole point: without it this is +# just a slower way of writing an exemption that never expires. +wd_mute_add() { + local ctr="$1" dur="$2" reason="${3:-}" + [[ -n "$ctr" && -n "$dur" ]] || { echo "usage: wd_mute_add <30m|2h|1d> [reason]" >&2; return 1; } + + local n unit secs + n="${dur%[mhd]}"; unit="${dur##*[0-9]}" + [[ "$n" =~ ^[0-9]+$ && "$n" -gt 0 ]] || { echo "bad duration: $dur" >&2; return 1; } + case "$unit" in + m) secs=$(( n * 60 )) ;; + h) secs=$(( n * 3600 )) ;; + d) secs=$(( n * 86400 )) ;; + *) echo "bad duration unit: $dur (use m, h or d)" >&2; return 1 ;; + esac + + local max=$(( ${WATCHDOG_MUTE_MAX_HOURS:-8} * 3600 )) + if (( secs > max )); then + echo "refused: ${dur} exceeds WATCHDOG_MUTE_MAX_HOURS=${WATCHDOG_MUTE_MAX_HOURS:-8}" >&2 + return 1 + fi + + # A pipe would split the record and a newline would forge one, so neither is allowed through. + reason="${reason//|/ }"; reason="${reason//$'\n'/ }" + + local f; f="$(_wd_mute_file)" + mkdir -p "$(dirname "$f")" 2>/dev/null + touch "$f" 2>/dev/null || { echo "cannot write $f" >&2; return 1; } + wd_mute_remove "$ctr" >/dev/null 2>&1 # re-muting replaces rather than stacks + printf '%s|%s|%s\n' "$ctr" "$(( $(date +%s) + secs ))" "$reason" >> "$f" + wd_mute_prune +} + +wd_mute_remove() { + local ctr="$1" f; f="$(_wd_mute_file)" + [[ -n "$ctr" && -w "$f" ]] || return 0 + local tmp="${f}.tmp.$$" + awk -F'|' -v c="$ctr" '$1!=c' "$f" > "$tmp" 2>/dev/null && mv -f "$tmp" "$f" + rm -f "$tmp" 2>/dev/null +} + # Internal — lock file path for this script _lock_file() { echo "$LOCK_DIR/${1:-$(_lock_name)}.lock"