Add timed mutes so a temporary problem stops needing a permanent exemption
Every existing exemption lasts until someone remembers to undo it, and nobody does — Healarr has sat in a pressure list since it was uninstalled and seven ignore entries name containers that are gone. A mute states when it ends and then ends, capped by WATCHDOG_MUTE_MAX_HOURS so temporary is enforced rather than intended. Applied where IGNORE_MAP is built, so all five check sites inherit it, and shown with its countdown because an invisible suppression is the thing being fixed.
This commit is contained in:
@@ -270,6 +270,18 @@
|
||||
# resumes on the next cycle. The container is not started — it remains stopped
|
||||
# until started manually.
|
||||
#
|
||||
# docker_watchdog.sh --mute ContainerName 2h "reason"
|
||||
# Silence every check for ContainerName until the time is up, then resume on its own.
|
||||
# Strikes, restarts, unhealthy and OOM reports and notifications are all suppressed —
|
||||
# the same suppression WATCHDOG_SCAN_IGNORE gives, with an end to it.
|
||||
# Duration is 30m, 2h or 1d, capped by WATCHDOG_MUTE_MAX_HOURS. Re-muting replaces.
|
||||
#
|
||||
# Use this, not --pause, when the container is meant to come back: --pause has no
|
||||
# expiry, which is how an exemption for one afternoon is still there a year later.
|
||||
#
|
||||
# docker_watchdog.sh --unmute ContainerName
|
||||
# End a mute early. Monitoring resumes on the next cycle.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -285,6 +297,10 @@ for (( _wdi=0; _wdi<${#PARSED_ARGS[@]}; _wdi++ )); do
|
||||
case "${PARSED_ARGS[$_wdi]}" in
|
||||
--pause) ((_wdi++)); WATCHDOG_PAUSE_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;;
|
||||
--resume) ((_wdi++)); WATCHDOG_RESUME_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;;
|
||||
--mute) ((_wdi++)); WATCHDOG_MUTE_CONTAINER="${PARSED_ARGS[$_wdi]:-}"
|
||||
((_wdi++)); WATCHDOG_MUTE_DURATION="${PARSED_ARGS[$_wdi]:-}"
|
||||
((_wdi++)); WATCHDOG_MUTE_REASON="${PARSED_ARGS[$_wdi]:-}" ;;
|
||||
--unmute) ((_wdi++)); WATCHDOG_UNMUTE_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;;
|
||||
esac
|
||||
done
|
||||
unset _wdi
|
||||
@@ -336,6 +352,13 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}"
|
||||
_intentional=$(cat "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
|
||||
echo "$ICON_SKIP Intentional: ${_intentional:-none}"
|
||||
_mutes=""
|
||||
while read -r _m; do
|
||||
[[ -n "$_m" ]] || continue
|
||||
_r=$(wd_mute_remaining "$_m")
|
||||
_mutes+="${_m}($(( ${_r:-0} / 60 ))m) "
|
||||
done < <(wd_mute_active)
|
||||
echo "$ICON_SKIP Muted: ${_mutes:-none}"
|
||||
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
|
||||
echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
|
||||
echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)"
|
||||
@@ -369,6 +392,31 @@ if [[ -n "$WATCHDOG_PAUSE_CONTAINER" || -n "$WATCHDOG_RESUME_CONTAINER" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Timed mutes — --mute / --unmute ──────────────────────────────────────────────────────────
|
||||
# Distinct from --pause, and the difference is the point. An intentional stop says "this is meant
|
||||
# to be down, leave it alone" and lasts until it is cleared. A mute says "leave it alone until
|
||||
# quarter past four", and then stops on its own — which is what a rebuild, a migration or a
|
||||
# vendor's broken update actually needs, and what nobody remembers to undo.
|
||||
if [[ -n "${WATCHDOG_MUTE_CONTAINER:-}" || -n "${WATCHDOG_UNMUTE_CONTAINER:-}" ]]; then
|
||||
if [[ -n "${WATCHDOG_MUTE_CONTAINER:-}" ]]; then
|
||||
if [[ -z "${WATCHDOG_MUTE_DURATION:-}" ]]; then
|
||||
error "--mute needs a duration: --mute <container> <30m|2h|1d> [reason]"
|
||||
exit 1
|
||||
fi
|
||||
if wd_mute_add "$WATCHDOG_MUTE_CONTAINER" "$WATCHDOG_MUTE_DURATION" "${WATCHDOG_MUTE_REASON:-}"; then
|
||||
_left=$(wd_mute_remaining "$WATCHDOG_MUTE_CONTAINER")
|
||||
success "$WATCHDOG_MUTE_CONTAINER muted for ${WATCHDOG_MUTE_DURATION} — expires $(date -d "@$(( $(date +%s) + ${_left:-0} ))" '+%H:%M' 2>/dev/null)"
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
if [[ -n "${WATCHDOG_UNMUTE_CONTAINER:-}" ]]; then
|
||||
wd_mute_remove "$WATCHDOG_UNMUTE_CONTAINER"
|
||||
success "$WATCHDOG_UNMUTE_CONTAINER unmuted — normal monitoring resumes next cycle"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
@@ -707,6 +755,14 @@ CYCLE_START=$(date +%s)
|
||||
for c in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do
|
||||
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
|
||||
done
|
||||
# A mute is a time-boxed ignore entry, so it is applied where the ignore map is built rather
|
||||
# than at each of the five places that consult it. Every existing check — strikes, restarts,
|
||||
# unhealthy, OOM, dependencies — inherits it without being touched, and nothing can be added
|
||||
# later that respects the ignore list but silently misses mutes.
|
||||
while read -r _muted; do
|
||||
[[ -n "$_muted" ]] && IGNORE_MAP["$_muted"]=1
|
||||
done < <(wd_mute_active)
|
||||
unset _muted
|
||||
|
||||
# ── Skip list and intentional stops visibility ───────────────────────────────────────────
|
||||
# Prune entries for containers that no longer exist at all (uninstalled/removed) from both
|
||||
|
||||
Reference in New Issue
Block a user