Add timed mutes so a temporary problem stops needing a permanent exemption

Every existing exemption lasts until someone remembers to undo it, and nobody
does — Healarr has sat in a pressure list since it was uninstalled and seven
ignore entries name containers that are gone. A mute states when it ends and
then ends, capped by WATCHDOG_MUTE_MAX_HOURS so temporary is enforced rather
than intended. Applied where IGNORE_MAP is built, so all five check sites
inherit it, and shown with its countdown because an invisible suppression is
the thing being fixed.
This commit is contained in:
Gmer4Lfe
2026-08-14 16:28:59 -04:00
parent 9f32644c32
commit 2cd1384786
5 changed files with 204 additions and 0 deletions
+56
View File
@@ -270,6 +270,18 @@
# resumes on the next cycle. The container is not started — it remains stopped
# until started manually.
#
# docker_watchdog.sh --mute ContainerName 2h "reason"
# Silence every check for ContainerName until the time is up, then resume on its own.
# Strikes, restarts, unhealthy and OOM reports and notifications are all suppressed —
# the same suppression WATCHDOG_SCAN_IGNORE gives, with an end to it.
# Duration is 30m, 2h or 1d, capped by WATCHDOG_MUTE_MAX_HOURS. Re-muting replaces.
#
# Use this, not --pause, when the container is meant to come back: --pause has no
# expiry, which is how an exemption for one afternoon is still there a year later.
#
# docker_watchdog.sh --unmute ContainerName
# End a mute early. Monitoring resumes on the next cycle.
#
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -285,6 +297,10 @@ for (( _wdi=0; _wdi<${#PARSED_ARGS[@]}; _wdi++ )); do
case "${PARSED_ARGS[$_wdi]}" in
--pause) ((_wdi++)); WATCHDOG_PAUSE_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;;
--resume) ((_wdi++)); WATCHDOG_RESUME_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;;
--mute) ((_wdi++)); WATCHDOG_MUTE_CONTAINER="${PARSED_ARGS[$_wdi]:-}"
((_wdi++)); WATCHDOG_MUTE_DURATION="${PARSED_ARGS[$_wdi]:-}"
((_wdi++)); WATCHDOG_MUTE_REASON="${PARSED_ARGS[$_wdi]:-}" ;;
--unmute) ((_wdi++)); WATCHDOG_UNMUTE_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;;
esac
done
unset _wdi
@@ -336,6 +352,13 @@ if [[ "$SHOW_STATUS" == true ]]; then
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}"
_intentional=$(cat "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
echo "$ICON_SKIP Intentional: ${_intentional:-none}"
_mutes=""
while read -r _m; do
[[ -n "$_m" ]] || continue
_r=$(wd_mute_remaining "$_m")
_mutes+="${_m}($(( ${_r:-0} / 60 ))m) "
done < <(wd_mute_active)
echo "$ICON_SKIP Muted: ${_mutes:-none}"
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)"
@@ -369,6 +392,31 @@ if [[ -n "$WATCHDOG_PAUSE_CONTAINER" || -n "$WATCHDOG_RESUME_CONTAINER" ]]; then
exit 0
fi
# ── Timed mutes — --mute / --unmute ──────────────────────────────────────────────────────────
# Distinct from --pause, and the difference is the point. An intentional stop says "this is meant
# to be down, leave it alone" and lasts until it is cleared. A mute says "leave it alone until
# quarter past four", and then stops on its own — which is what a rebuild, a migration or a
# vendor's broken update actually needs, and what nobody remembers to undo.
if [[ -n "${WATCHDOG_MUTE_CONTAINER:-}" || -n "${WATCHDOG_UNMUTE_CONTAINER:-}" ]]; then
if [[ -n "${WATCHDOG_MUTE_CONTAINER:-}" ]]; then
if [[ -z "${WATCHDOG_MUTE_DURATION:-}" ]]; then
error "--mute needs a duration: --mute <container> <30m|2h|1d> [reason]"
exit 1
fi
if wd_mute_add "$WATCHDOG_MUTE_CONTAINER" "$WATCHDOG_MUTE_DURATION" "${WATCHDOG_MUTE_REASON:-}"; then
_left=$(wd_mute_remaining "$WATCHDOG_MUTE_CONTAINER")
success "$WATCHDOG_MUTE_CONTAINER muted for ${WATCHDOG_MUTE_DURATION} — expires $(date -d "@$(( $(date +%s) + ${_left:-0} ))" '+%H:%M' 2>/dev/null)"
else
exit 1
fi
fi
if [[ -n "${WATCHDOG_UNMUTE_CONTAINER:-}" ]]; then
wd_mute_remove "$WATCHDOG_UNMUTE_CONTAINER"
success "$WATCHDOG_UNMUTE_CONTAINER unmuted — normal monitoring resumes next cycle"
fi
exit 0
fi
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -707,6 +755,14 @@ CYCLE_START=$(date +%s)
for c in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
done
# A mute is a time-boxed ignore entry, so it is applied where the ignore map is built rather
# than at each of the five places that consult it. Every existing check — strikes, restarts,
# unhealthy, OOM, dependencies — inherits it without being touched, and nothing can be added
# later that respects the ignore list but silently misses mutes.
while read -r _muted; do
[[ -n "$_muted" ]] && IGNORE_MAP["$_muted"]=1
done < <(wd_mute_active)
unset _muted
# ── Skip list and intentional stops visibility ───────────────────────────────────────────
# Prune entries for containers that no longer exist at all (uninstalled/removed) from both