docker_watchdog: skip required containers stopped cleanly or explicitly paused
Exit code 0 on a required container (docker stop, UI stop) now reads as intentional — no strike, no restart. Non-zero exits still trigger the existing strike → restart path. Adds --pause / --resume management commands and a persistent intentional- stops state file for maintenance windows where even the exit-code heuristic isn't enough. Containers auto-cleared from the list when seen running again.
This commit is contained in:
Gmer4Lfe
@@ -130,10 +130,11 @@
|
||||
# STATE FILES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots)
|
||||
# WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots)
|
||||
# DOCKER_WATCHDOG_FAILED_FILE — container skip list (STATE_DIR — survives reboots)
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR)
|
||||
# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag
|
||||
# DOCKER_WATCHDOG_INTENTIONAL_FILE — intentional stops list (STATE_DIR — survives reboots)
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR)
|
||||
# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
@@ -198,6 +199,10 @@
|
||||
# DOCKER_WATCHDOG_HEARTBEAT_HOURS
|
||||
# Hours between alive heartbeat log entries
|
||||
#
|
||||
# DOCKER_WATCHDOG_INTENTIONAL_FILE
|
||||
# Path to intentional stops state file (STATE_DIR). Containers in this file
|
||||
# are never restarted by the watchdog, regardless of exit code.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
@@ -210,13 +215,23 @@
|
||||
# happen based on current container states. Use to verify configuration.
|
||||
#
|
||||
# docker_watchdog.sh --status
|
||||
# Show strike counts, skip list contents, grace period status, RAM emergency
|
||||
# deferral state, and last cycle timing. Then exit.
|
||||
# Show strike counts, skip list contents, intentional stops, grace period status,
|
||||
# RAM emergency deferral state, and last cycle timing. Then exit.
|
||||
#
|
||||
# docker_watchdog.sh --log
|
||||
# Verbose output — full detail for every container checked and every decision.
|
||||
# Use to debug why a container is or is not being restarted.
|
||||
#
|
||||
# docker_watchdog.sh --pause ContainerName
|
||||
# Add ContainerName to the intentional stops list. Watchdog will not strike or
|
||||
# restart it until it is seen running again or --resume is called. Persists across
|
||||
# reboots. Use when stopping a required container for planned maintenance.
|
||||
#
|
||||
# docker_watchdog.sh --resume ContainerName
|
||||
# Remove ContainerName from the intentional stops list. Normal watchdog monitoring
|
||||
# resumes on the next cycle. The container is not started — it remains stopped
|
||||
# until started manually.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -225,6 +240,17 @@ source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# Extract watchdog-specific flags not handled by common parse_args
|
||||
WATCHDOG_PAUSE_CONTAINER=""
|
||||
WATCHDOG_RESUME_CONTAINER=""
|
||||
for (( _wdi=0; _wdi<${#PARSED_ARGS[@]}; _wdi++ )); do
|
||||
case "${PARSED_ARGS[$_wdi]}" in
|
||||
--pause) ((_wdi++)); WATCHDOG_PAUSE_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;;
|
||||
--resume) ((_wdi++)); WATCHDOG_RESUME_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;;
|
||||
esac
|
||||
done
|
||||
unset _wdi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup — runs once at start ━━━
|
||||
# ==============================================================================================
|
||||
@@ -256,7 +282,7 @@ log "$ICON_CONTAINERS Tier1: watched=${#WATCHDOG_CONTAINERS[@]} required=${#WAT
|
||||
|
||||
# Ensure state files exist
|
||||
touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \
|
||||
"$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
"$DOCKER_WATCHDOG_FAILED_FILE" "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null
|
||||
|
||||
# Timeout for all docker commands — configurable via WATCHDOG_DAEMON_TIMEOUT in master.conf
|
||||
DOCKER_TIMEOUT="${WATCHDOG_DAEMON_TIMEOUT:-20}"
|
||||
@@ -270,6 +296,8 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
_watched="${!WATCHDOG_CONTAINERS[*]}"; echo "$ICON_CONTAINERS Watched: ${_watched:-none}"
|
||||
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}"
|
||||
_intentional=$(cat "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
|
||||
echo "$ICON_SKIP Intentional: ${_intentional:-none}"
|
||||
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
|
||||
echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
|
||||
echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)"
|
||||
@@ -286,6 +314,23 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Intentional stop management — --pause / --resume ─────────────────────────────────────────
|
||||
if [[ -n "$WATCHDOG_PAUSE_CONTAINER" || -n "$WATCHDOG_RESUME_CONTAINER" ]]; then
|
||||
if [[ -n "$WATCHDOG_PAUSE_CONTAINER" ]]; then
|
||||
if grep -q "^${WATCHDOG_PAUSE_CONTAINER}$" "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null; then
|
||||
warn "$WATCHDOG_PAUSE_CONTAINER already in intentional stops"
|
||||
else
|
||||
echo "$WATCHDOG_PAUSE_CONTAINER" >> "$DOCKER_WATCHDOG_INTENTIONAL_FILE"
|
||||
success "$WATCHDOG_PAUSE_CONTAINER added to intentional stops — watchdog will not restart it"
|
||||
fi
|
||||
fi
|
||||
if [[ -n "$WATCHDOG_RESUME_CONTAINER" ]]; then
|
||||
sed -i "/^${WATCHDOG_RESUME_CONTAINER}$/d" "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null
|
||||
success "$WATCHDOG_RESUME_CONTAINER removed from intentional stops — normal monitoring resumes next cycle"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
@@ -326,6 +371,19 @@ remove_from_skip_list() {
|
||||
warn "$1 recovered — removed from skip list ✅"
|
||||
}
|
||||
|
||||
# Check if container is in the intentional stops list
|
||||
is_intentional_stop() {
|
||||
grep -q "^${1}$" "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null
|
||||
}
|
||||
|
||||
# Remove from intentional stops — called when container is seen running again
|
||||
clear_intentional_stop() {
|
||||
if grep -q "^${1}$" "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null; then
|
||||
sed -i "/^${1}$/d" "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null
|
||||
warn "$1 running — removed from intentional stops ✅"
|
||||
fi
|
||||
}
|
||||
|
||||
# Log a restart event to the rolling restart history file
|
||||
log_restart() {
|
||||
local container="$1"
|
||||
@@ -616,9 +674,11 @@ CYCLE_START=$(date +%s)
|
||||
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
|
||||
done
|
||||
|
||||
# ── Skip list visibility ─────────────────────────────────────────────────────────────────
|
||||
# ── Skip list and intentional stops visibility ───────────────────────────────────────────
|
||||
_skip_contents=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
|
||||
[[ -n "$_skip_contents" ]] && warn "$ICON_SKIP Skip list active: $_skip_contents — manual intervention needed"
|
||||
_intentional_contents=$(cat "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
|
||||
[[ -n "$_intentional_contents" ]] && warn "$ICON_SKIP Intentional stops: $_intentional_contents — watchdog will not restart these"
|
||||
|
||||
# ── Docker daemon health check — first check every run ──────────────────────────────────
|
||||
# If daemon is hung all container operations will fail — check first, skip run if down
|
||||
@@ -676,23 +736,41 @@ CYCLE_START=$(date +%s)
|
||||
fi
|
||||
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
# Running — clear any strikes
|
||||
# Running — clear strikes and any intentional-stop flag set from a prior cycle
|
||||
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
clear_intentional_stop "$container"
|
||||
log "$ICON_RUNNING $container — running ✅"
|
||||
else
|
||||
# Check intentional stops first — explicit operator instruction beats everything
|
||||
if is_intentional_stop "$container"; then
|
||||
log "$container — intentionally stopped (on pause list) — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Exit code 0 = cleanly stopped (docker stop, Unraid UI stop, clean shutdown).
|
||||
# Don't strike or restart — operator almost certainly stopped it on purpose.
|
||||
# Use --pause to make this permanent across reboots.
|
||||
LAST_EXIT=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.ExitCode}}' "$container" 2>/dev/null)
|
||||
if [[ "$LAST_EXIT" == "0" ]]; then
|
||||
log "$container — stopped cleanly (exit 0) — treating as intentional; use --pause to suppress permanently"
|
||||
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
continue
|
||||
fi
|
||||
|
||||
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
|
||||
STRIKES=$(( STRIKES + 1 ))
|
||||
set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
|
||||
warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
|
||||
warn "$container — not running, exit ${LAST_EXIT} (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
|
||||
((T1_WARNINGS++))
|
||||
|
||||
if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
|
||||
result=0
|
||||
safe_restart "$container" "required container down" || result=$?
|
||||
safe_restart "$container" "required container down (exit ${LAST_EXIT})" || result=$?
|
||||
case $result in
|
||||
0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container was down and restarted on $(hostname)" "warning" ;;
|
||||
queue_notify "$container was down (exit ${LAST_EXIT}) and restarted on $(hostname)" "warning" ;;
|
||||
2) : ;; # Added to skip list — already notified
|
||||
*) queue_notify "$container failed to restart on $(hostname)" "warning" ;;
|
||||
esac
|
||||
|
||||
Reference in New Issue
Block a user