added a global check system for docker to docker watchdog
This commit is contained in:
@@ -2,28 +2,35 @@
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Docker Watchdog --------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# First line of defense — monitors Docker containers for memory, CPU, HTTP responsiveness,
|
||||
# and unexpected stops. Restarts containers that exceed thresholds or go offline.
|
||||
# Two-tier self-healing container monitoring system.
|
||||
#
|
||||
# Works alongside system_watchdog.sh:
|
||||
# docker_watchdog.sh — container level, minimal disruption, tries to self-heal
|
||||
# system_watchdog.sh — system level, last resort, reboots when healing fails
|
||||
# Tier 1 — Strict monitoring (configured containers only)
|
||||
# Memory hard limits — immediate restart if exceeded
|
||||
# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT strikes
|
||||
# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT strikes
|
||||
# Required containers — must always be running, strike system with skip list
|
||||
#
|
||||
# Behaviour:
|
||||
# Memory — immediate restart if hard limit exceeded
|
||||
# CPU — strike system, restarts after CPU_FAIL_LIMIT consecutive hits
|
||||
# HTTP — strike system, restarts after RESP_FAIL_LIMIT consecutive failures
|
||||
# Required — strike system, restarts stopped containers, persistent skip list
|
||||
# prevents reboot loops, auto-clears when container recovers
|
||||
# Daemon — immediate notify if Docker daemon is unresponsive
|
||||
# Tier 2 — Global health scan (all running containers)
|
||||
# Unhealthy status — Docker HEALTHCHECK unhealthy → restart
|
||||
# OOM killed — kernel killed container → restart + notify
|
||||
# Crash loop detection — RestartCount climbing → notify, critical above limit
|
||||
# Dead containers — remove and restart
|
||||
# Unexpected exits — non-zero exit code → restart
|
||||
#
|
||||
# Strike cadence depends on cron schedule:
|
||||
# Every 15min + 2 strikes = 30min sustained before restart
|
||||
# Every 10min + 2 strikes = 20min sustained before restart
|
||||
# Every 5min + 2 strikes = 10min sustained before restart
|
||||
# Cross-cutting intelligence (applies to both tiers):
|
||||
# Startup grace period — skip restarts while system is still booting
|
||||
# Dependency ordering — restart database before app, not the other way around
|
||||
# Restart loop protect — stop restarting after X restarts in X hours → skip list
|
||||
# Skip list auto-clear — clears when container recovers, fresh start
|
||||
# Notification batching — one clean summary per run, not one ping per event
|
||||
#
|
||||
# State files:
|
||||
# WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot)
|
||||
# SYS_WATCHDOG_FAILED_FILE — persistent skip list (/boot — survives reboots)
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (/boot)
|
||||
#
|
||||
# All configuration in Master.conf under Docker Watchdog section.
|
||||
# Supports --dry-run to show what would happen without acting.
|
||||
# Supports --dry-run and --status.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -33,12 +40,6 @@ source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
TOTAL_CORES=$(nproc)
|
||||
|
||||
# Persistent skip list — shared with system_watchdog.sh
|
||||
# Containers in this list are skipped until they recover
|
||||
SKIP_LIST_FILE="$SYS_WATCHDOG_FAILED_FILE"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
@@ -51,17 +52,27 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
info "$ICON_WATCHDOG Watchdog initialising — $TOTAL_CORES cores detected"
|
||||
|
||||
touch "$WATCHDOG_STATE_FILE" 2>/dev/null || {
|
||||
error "Cannot create state file: $WATCHDOG_STATE_FILE"
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
error "Docker not found"
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
|
||||
touch "$SKIP_LIST_FILE" 2>/dev/null || {
|
||||
error "Cannot create skip list: $SKIP_LIST_FILE"
|
||||
exit 1
|
||||
}
|
||||
success "Docker found"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||
|
||||
# State files
|
||||
touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \
|
||||
"$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
|
||||
# Build ignore map for Tier 2
|
||||
declare -A IGNORE_MAP
|
||||
for c in "${WATCHDOG_SCAN_IGNORE[@]}"; do
|
||||
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
|
||||
done
|
||||
|
||||
# Batch notification collector
|
||||
NOTIFY_EVENTS=()
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
@@ -69,378 +80,577 @@ touch "$SKIP_LIST_FILE" 2>/dev/null || {
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_WATCHDOG Monitored: ${!WATCHDOG_CONTAINERS[*]}"
|
||||
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}"
|
||||
echo "$ICON_MEM Soft mem: ${SOFT_MEM_THRESHOLD}% of limit"
|
||||
echo "$ICON_ZFS CPU soft: ${SOFT_CPU_THRESHOLD}%"
|
||||
echo "$ICON_ZFS CPU hard: ${HARD_CPU_THRESHOLD}%"
|
||||
echo "$ICON_RETRY CPU strikes: ${CPU_FAIL_LIMIT}"
|
||||
echo "$ICON_RETRY Resp strikes: ${RESP_FAIL_LIMIT}"
|
||||
echo "$ICON_TIME Curl timeout: ${CURL_TIMEOUT}s"
|
||||
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_CONTAINERS Watched: ${!WATCHDOG_CONTAINERS[@]}"
|
||||
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}"
|
||||
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
|
||||
echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s"
|
||||
echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
|
||||
echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY"
|
||||
echo "$ICON_WATCHDOG Ignore list: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
|
||||
echo "$ICON_TIME System uptime: $(format_duration $UPTIME_S)"
|
||||
[[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]] && \
|
||||
warn "Within startup grace period — restarts suppressed"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# STATE HELPERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
get_strikes() {
|
||||
local container="$1" metric="$2"
|
||||
grep -E "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f3
|
||||
}
|
||||
|
||||
set_strikes() {
|
||||
local container="$1" metric="$2" count="$3"
|
||||
grep -vE "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null > "${WATCHDOG_STATE_FILE}.tmp"
|
||||
echo "${container}:${metric}:${count}" >> "${WATCHDOG_STATE_FILE}.tmp"
|
||||
mv "${WATCHDOG_STATE_FILE}.tmp" "$WATCHDOG_STATE_FILE"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# SKIP LIST HELPERS
|
||||
# Container skip list — persistent across reboots via /boot/
|
||||
# Auto-clears entries when container is found running again.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
is_in_skip_list() {
|
||||
local container="$1"
|
||||
grep -qE "^${container}$" "$SKIP_LIST_FILE" 2>/dev/null
|
||||
}
|
||||
|
||||
add_to_skip_list() {
|
||||
local container="$1"
|
||||
if ! is_in_skip_list "$container"; then
|
||||
echo "$container" >> "$SKIP_LIST_FILE"
|
||||
warn "$ICON_WATCHDOG $container added to persistent skip list"
|
||||
notify "$container added to watchdog skip list on $(hostname) — manual check recommended" "Docker Watchdog" "warning"
|
||||
fi
|
||||
}
|
||||
|
||||
remove_from_skip_list() {
|
||||
local container="$1"
|
||||
grep -vE "^${container}$" "$SKIP_LIST_FILE" 2>/dev/null > "${SKIP_LIST_FILE}.tmp"
|
||||
mv "${SKIP_LIST_FILE}.tmp" "$SKIP_LIST_FILE"
|
||||
success "$ICON_WATCHDOG $container recovered — removed from skip list"
|
||||
notify "$container recovered and removed from watchdog skip list on $(hostname)" "Docker Watchdog" "normal"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# SKIP LIST AUTO-HEAL CHECK
|
||||
# On every run check if any skipped containers are now running.
|
||||
# If running remove from skip list — could have recovered after reboot or manual fix.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_skip_list_recovery() {
|
||||
[[ ! -s "$SKIP_LIST_FILE" ]] && return
|
||||
|
||||
info "$ICON_WATCHDOG Checking skip list for recovered containers..."
|
||||
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
remove_from_skip_list "$container"
|
||||
else
|
||||
log "$container still not running — remains on skip list"
|
||||
fi
|
||||
done < "$SKIP_LIST_FILE"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# DOCKER DAEMON HEALTH CHECK
|
||||
# Verifies Docker daemon is responding before attempting any container operations.
|
||||
# A hung daemon means all checks will fail — notify immediately and exit.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_docker_daemon() {
|
||||
info "$ICON_CONTAINERS Checking Docker daemon..."
|
||||
|
||||
if ! timeout 10 docker ps >/dev/null 2>&1; then
|
||||
error "Docker daemon is not responding"
|
||||
notify "Docker daemon unresponsive on $(hostname) — immediate attention required" "Docker Watchdog" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Docker daemon is healthy"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# HELPERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
parse_stats() {
|
||||
local container="$1"
|
||||
docker stats --no-stream --format "{{.MemUsage}}|{{.CPUPerc}}" "$container"
|
||||
# Strike count management
|
||||
get_strikes() {
|
||||
grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"
|
||||
}
|
||||
|
||||
convert_to_mb() {
|
||||
local value="$1" unit="$2"
|
||||
case "$unit" in
|
||||
KiB) awk "BEGIN {print $value / 1024}" ;;
|
||||
MiB) echo "$value" ;;
|
||||
GiB) awk "BEGIN {print $value * 1024}" ;;
|
||||
*) echo "UNKNOWN" ;;
|
||||
esac
|
||||
set_strikes() {
|
||||
local container="$1" count="$2" file="$3"
|
||||
if grep -q "^${container}:" "$file" 2>/dev/null; then
|
||||
sed -i "s/^${container}:.*/${container}:${count}/" "$file"
|
||||
else
|
||||
echo "${container}:${count}" >> "$file"
|
||||
fi
|
||||
}
|
||||
|
||||
# Restarts a container and sends notification.
|
||||
# Failed restarts also notify — system_watchdog.sh is the next line of defense.
|
||||
restart_container() {
|
||||
# Persistent skip list management
|
||||
is_skipped() {
|
||||
grep -q "^${1}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
}
|
||||
|
||||
add_to_skip_list() {
|
||||
local container="$1" reason="$2"
|
||||
if ! is_skipped "$container"; then
|
||||
echo "$container" >> "$SYS_WATCHDOG_FAILED_FILE"
|
||||
error "$container added to skip list — $reason"
|
||||
queue_notify "$container added to skip list on $(hostname) — $reason — manual intervention needed" "critical"
|
||||
fi
|
||||
}
|
||||
|
||||
remove_from_skip_list() {
|
||||
sed -i "/^${1}$/d" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
success "$1 removed from skip list — recovered"
|
||||
}
|
||||
|
||||
# Restart loop tracking — /boot/ bounded file, auto-purges old entries
|
||||
log_restart() {
|
||||
local container="$1"
|
||||
local now
|
||||
now=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
local cutoff
|
||||
cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# Append this restart
|
||||
echo "${container}|${now}" >> "$WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
|
||||
# Purge entries older than window — keeps file bounded
|
||||
local tmp="${WATCHDOG_CONTAINER_RESTART_LOG}.tmp"
|
||||
awk -F'|' -v cutoff="$cutoff" '$2 >= cutoff' \
|
||||
"$WATCHDOG_CONTAINER_RESTART_LOG" > "$tmp" && \
|
||||
mv "$tmp" "$WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
}
|
||||
|
||||
# Count restarts for a container within the current window
|
||||
get_restart_count() {
|
||||
local container="$1"
|
||||
local cutoff
|
||||
cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S')
|
||||
awk -F'|' -v c="$container" -v cutoff="$cutoff" \
|
||||
'$1==c && $2>=cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l
|
||||
}
|
||||
|
||||
# Dependency check — returns 0 if all dependencies are running
|
||||
dependencies_satisfied() {
|
||||
local container="$1"
|
||||
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
|
||||
[[ -z "$deps" ]] && return 0
|
||||
|
||||
for dep in $deps; do
|
||||
local status
|
||||
status=$(docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null)
|
||||
if [[ "$status" != "true" ]]; then
|
||||
warn "$container — dependency $dep is not running — skipping restart this cycle"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# Restart a container with restart loop protection
|
||||
# Returns 0 on success, 1 on failure, 2 if loop limit hit
|
||||
safe_restart() {
|
||||
local container="$1" reason="$2"
|
||||
|
||||
# Check restart loop
|
||||
local restart_count
|
||||
restart_count=$(get_restart_count "$container")
|
||||
|
||||
if [[ "$restart_count" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then
|
||||
add_to_skip_list "$container" \
|
||||
"restarted $restart_count times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
|
||||
return 2
|
||||
fi
|
||||
|
||||
# Check dependencies
|
||||
dependencies_satisfied "$container" || return 1
|
||||
|
||||
# Check startup grace
|
||||
local uptime_s
|
||||
uptime_s=$(awk '{print int($1)}' /proc/uptime)
|
||||
if [[ "$uptime_s" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||
warn "$container — within startup grace period (${uptime_s}s / ${WATCHDOG_STARTUP_GRACE}s) — skipping restart"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $container ($reason)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
info "Restarting $container ($reason)..."
|
||||
info "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]..."
|
||||
if docker restart "$container" >/dev/null 2>&1; then
|
||||
echo "$ICON_STARTED $container restarted"
|
||||
notify "$container restarted on $(hostname) — $reason" "Docker Watchdog" "warning"
|
||||
success "$ICON_STARTED $container restarted"
|
||||
log_restart "$container"
|
||||
return 0
|
||||
else
|
||||
error "Failed to restart $container"
|
||||
notify "Failed to restart $container on $(hostname) — $reason" "Docker Watchdog" "warning"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# MEMORY CHECK
|
||||
# Restarts immediately if container exceeds hard memory limit.
|
||||
# Warns if approaching soft threshold.
|
||||
# Usage: check_memory "Emby" 16384
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_memory() {
|
||||
local container="$1" limit_mb="$2"
|
||||
local stats mem_raw mem_val mem_unit mem_mb usage_pct
|
||||
# Queue a notification event for batching
|
||||
queue_notify() {
|
||||
local message="$1" severity="${2:-warning}"
|
||||
NOTIFY_EVENTS+=("${severity}|${message}")
|
||||
log "Queued: $message"
|
||||
}
|
||||
|
||||
stats=$(parse_stats "$container")
|
||||
mem_raw=$(echo "$stats" | awk -F'|' '{print $1}' | awk '{print $1}')
|
||||
mem_val=$(echo "$mem_raw" | sed -E 's/([0-9.]+).*/\1/')
|
||||
mem_unit=$(echo "$mem_raw" | sed -E 's/[0-9.]+([a-zA-Z]+).*/\1/')
|
||||
mem_mb=$(convert_to_mb "$mem_val" "$mem_unit")
|
||||
local mem_int
|
||||
mem_int=$(printf "%.0f" "$mem_mb")
|
||||
usage_pct=$(( (mem_int * 100) / limit_mb ))
|
||||
# Send all queued notifications
|
||||
flush_notify() {
|
||||
[[ ${#NOTIFY_EVENTS[@]} -eq 0 ]] && return
|
||||
|
||||
if (( usage_pct >= 100 )); then
|
||||
error "$ICON_MEM $container memory ${mem_int}MB (${usage_pct}%) — exceeded ${limit_mb}MB hard limit"
|
||||
restart_container "$container" "memory hard limit"
|
||||
set_strikes "$container" "CPU" 0
|
||||
set_strikes "$container" "RESP" 0
|
||||
elif (( usage_pct >= SOFT_MEM_THRESHOLD )); then
|
||||
warn "$ICON_MEM $container memory ${mem_int}MB (${usage_pct}% of ${limit_mb}MB limit)"
|
||||
if [[ "$WATCHDOG_BATCH_NOTIFY" == "true" ]]; then
|
||||
# Build single batched message
|
||||
local critical_count=0 warning_count=0
|
||||
local messages=()
|
||||
local highest_severity="normal"
|
||||
|
||||
for event in "${NOTIFY_EVENTS[@]}"; do
|
||||
local sev="${event%%|*}"
|
||||
local msg="${event#*|}"
|
||||
messages+=("$msg")
|
||||
[[ "$sev" == "critical" ]] && ((critical_count++)) && highest_severity="warning"
|
||||
[[ "$sev" == "warning" ]] && ((warning_count++)) && \
|
||||
[[ "$highest_severity" == "normal" ]] && highest_severity="warning"
|
||||
done
|
||||
|
||||
local summary
|
||||
summary=$(printf '%s. ' "${messages[@]}")
|
||||
notify "Docker Watchdog on $(hostname) — ${#NOTIFY_EVENTS[@]} event(s): $summary" \
|
||||
"Docker Watchdog" "$highest_severity"
|
||||
else
|
||||
success "$ICON_MEM $container memory ${mem_int}MB (${usage_pct}% of ${limit_mb}MB limit)"
|
||||
# Send individual notifications
|
||||
for event in "${NOTIFY_EVENTS[@]}"; do
|
||||
local sev="${event%%|*}"
|
||||
local msg="${event#*|}"
|
||||
[[ "$sev" == "critical" ]] && sev="warning"
|
||||
notify "$msg" "Docker Watchdog" "$sev"
|
||||
done
|
||||
fi
|
||||
|
||||
NOTIFY_EVENTS=()
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# CPU CHECK
|
||||
# Strike system — restarts after CPU_FAIL_LIMIT consecutive over-threshold checks.
|
||||
# Resets strikes on recovery or restart.
|
||||
# Usage: check_cpu "Emby"
|
||||
# STARTUP GRACE CHECK
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_cpu() {
|
||||
local container="$1"
|
||||
local stats cpu_raw cpu_norm cpu_int violations
|
||||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||||
IN_GRACE_PERIOD=false
|
||||
|
||||
stats=$(parse_stats "$container")
|
||||
cpu_raw=$(echo "$stats" | awk -F'|' '{print $2}' | tr -d '%')
|
||||
cpu_norm=$(awk "BEGIN {print $cpu_raw / $TOTAL_CORES}")
|
||||
cpu_int=$(printf "%.0f" "$cpu_norm")
|
||||
|
||||
violations=$(get_strikes "$container" "CPU")
|
||||
[[ -z "$violations" ]] && violations=0
|
||||
|
||||
if (( cpu_int >= HARD_CPU_THRESHOLD )); then
|
||||
((violations++))
|
||||
error "$ICON_ZFS $container CPU ${cpu_int}% — hard threshold ($violations/$CPU_FAIL_LIMIT strikes)"
|
||||
set_strikes "$container" "CPU" "$violations"
|
||||
elif (( cpu_int >= SOFT_CPU_THRESHOLD )); then
|
||||
((violations++))
|
||||
warn "$ICON_ZFS $container CPU ${cpu_int}% — soft threshold ($violations/$CPU_FAIL_LIMIT strikes)"
|
||||
set_strikes "$container" "CPU" "$violations"
|
||||
else
|
||||
[[ $violations -gt 0 ]] && info "$ICON_ZFS $container CPU ${cpu_int}% — recovered, resetting strikes"
|
||||
[[ $violations -eq 0 ]] && success "$ICON_ZFS $container CPU ${cpu_int}%"
|
||||
set_strikes "$container" "CPU" 0
|
||||
violations=0
|
||||
fi
|
||||
|
||||
if (( violations >= CPU_FAIL_LIMIT )); then
|
||||
error "$ICON_ZFS $container CPU limit hit for $CPU_FAIL_LIMIT consecutive checks"
|
||||
restart_container "$container" "sustained CPU abuse"
|
||||
set_strikes "$container" "CPU" 0
|
||||
fi
|
||||
}
|
||||
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||
GRACE_REMAINING=$(( WATCHDOG_STARTUP_GRACE - UPTIME_SECONDS ))
|
||||
warn "System uptime $(format_duration $UPTIME_SECONDS) — startup grace period active (${GRACE_REMAINING}s remaining)"
|
||||
warn "Checks will run but restarts are suppressed during grace period"
|
||||
IN_GRACE_PERIOD=true
|
||||
else
|
||||
info "System uptime $(format_duration $UPTIME_SECONDS) — grace period passed"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# RESPONSIVENESS CHECK
|
||||
# Strike system — restarts after RESP_FAIL_LIMIT consecutive failed HTTP checks.
|
||||
# Skips containers with no URL defined in WATCHDOG_CONTAINER_URLS.
|
||||
# Usage: check_responsiveness "Emby"
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# TIER 1 — Strict Monitoring
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_responsiveness() {
|
||||
local container="$1"
|
||||
local url="${WATCHDOG_CONTAINER_URLS[$container]:-}"
|
||||
|
||||
[[ -z "$url" ]] && return
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " $ICON_WATCHDOG TIER 1 — Strict Monitoring"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
local fails
|
||||
fails=$(get_strikes "$container" "RESP")
|
||||
[[ -z "$fails" ]] && fails=0
|
||||
T1_RESTARTS=0
|
||||
T1_WARNINGS=0
|
||||
|
||||
if ! curl -s --max-time "$CURL_TIMEOUT" "$url" >/dev/null 2>&1; then
|
||||
((fails++))
|
||||
warn "$ICON_PING $container unresponsive at $url ($fails/$RESP_FAIL_LIMIT strikes)"
|
||||
set_strikes "$container" "RESP" "$fails"
|
||||
else
|
||||
[[ $fails -gt 0 ]] && info "$ICON_PING $container responsive again — resetting strikes"
|
||||
[[ $fails -eq 0 ]] && success "$ICON_PING $container responsive at $url"
|
||||
set_strikes "$container" "RESP" 0
|
||||
fails=0
|
||||
fi
|
||||
|
||||
if (( fails >= RESP_FAIL_LIMIT )); then
|
||||
error "$ICON_PING $container unresponsive for $RESP_FAIL_LIMIT consecutive checks"
|
||||
restart_container "$container" "HTTP unresponsive"
|
||||
set_strikes "$container" "RESP" 0
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REQUIRED CONTAINER CHECK
|
||||
# Monitors WATCHDOG_REQUIRED_CONTAINERS for unexpected stops.
|
||||
# Strike system — attempts restart on each strike.
|
||||
# After strike limit hit — adds to persistent skip list and notifies system_watchdog handoff.
|
||||
# Skip list auto-clears at start of each run if container has recovered.
|
||||
# Usage: check_required_containers
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_required_containers() {
|
||||
[[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -eq 0 ]] && return
|
||||
|
||||
info "$ICON_CONTAINERS Checking required containers..."
|
||||
# ── Required Containers ──────────────────────────────────────────────────────────────────────
|
||||
if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Required Containers ━━━"
|
||||
|
||||
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
|
||||
# Skip if on persistent skip list
|
||||
if is_in_skip_list "$container"; then
|
||||
warn "$ICON_NOT_RUNNING $container is on skip list — skipping until recovered"
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
|
||||
# Check skip list
|
||||
if is_skipped "$container"; then
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
remove_from_skip_list "$container"
|
||||
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
# Clear restart history on recovery
|
||||
sed -i "/^${container}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
|
||||
queue_notify "$container recovered on $(hostname) — removed from skip list" "normal"
|
||||
else
|
||||
warn "$ICON_NOT_RUNNING $container — on skip list, manual intervention needed"
|
||||
RESTART_COUNT=$(get_restart_count "$container")
|
||||
warn " Restarted $RESTART_COUNT time(s) in last ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
success "$ICON_RUNNING $container is running"
|
||||
set_strikes "$container" "STOP" 0
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$STATUS" == "unknown" ]]; then
|
||||
warn "$container not found on this host — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Container is stopped — apply strike
|
||||
local strikes
|
||||
strikes=$(get_strikes "$container" "STOP")
|
||||
[[ -z "$strikes" ]] && strikes=0
|
||||
((strikes++))
|
||||
|
||||
warn "$ICON_NOT_RUNNING $container is stopped ($strikes/$SYS_WATCHDOG_STRIKE_LIMIT strikes)"
|
||||
set_strikes "$container" "STOP" "$strikes"
|
||||
|
||||
# Attempt restart on each strike
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would attempt restart of $container"
|
||||
success "$ICON_RUNNING $container — running"
|
||||
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
else
|
||||
if restart_container "$container" "unexpected stop"; then
|
||||
set_strikes "$container" "STOP" 0
|
||||
else
|
||||
# Restart failed
|
||||
if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then
|
||||
error "$container failed to restart after $SYS_WATCHDOG_STRIKE_LIMIT attempts"
|
||||
add_to_skip_list "$container"
|
||||
set_strikes "$container" "STOP" 0
|
||||
notify "$container handed off to system_watchdog on $(hostname) — added to skip list" "Docker Watchdog" "warning"
|
||||
fi
|
||||
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
|
||||
STRIKES=$(( STRIKES + 1 ))
|
||||
set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
|
||||
warn "$ICON_NOT_RUNNING $container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
|
||||
((T1_WARNINGS++))
|
||||
|
||||
if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
|
||||
result=0
|
||||
safe_restart "$container" "required container down" || result=$?
|
||||
case $result in
|
||||
0)
|
||||
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container was down and restarted on $(hostname)" "warning"
|
||||
;;
|
||||
2)
|
||||
: # already added to skip list in safe_restart
|
||||
;;
|
||||
*)
|
||||
queue_notify "$container failed to restart on $(hostname) — check dependencies and logs" "warning"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
done
|
||||
}
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_WATCHDOG Watchdog Run ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_WATCHDOG Watchdog Run — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
SKIPPED=()
|
||||
|
||||
# Daemon check first — if daemon is down nothing else works
|
||||
check_docker_daemon
|
||||
|
||||
# Auto-heal skip list before processing
|
||||
check_skip_list_recovery
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Resource monitoring — WATCHDOG_CONTAINERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── Memory and CPU Monitoring ────────────────────────────────────────────────────────────────
|
||||
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_MEM Resource Monitoring ━━━"
|
||||
echo "━━━ $ICON_MEM Memory / CPU Monitoring ━━━"
|
||||
|
||||
STATS=$(docker stats --no-stream \
|
||||
--format "{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}" 2>/dev/null)
|
||||
TOTAL_CORES=$(nproc 2>/dev/null || echo 1)
|
||||
|
||||
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
|
||||
echo ""
|
||||
info "$ICON_CONTAINERS $container"
|
||||
MEM_LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
|
||||
CONTAINER_STATS=$(echo "$STATS" | grep "^${container}|" | head -1)
|
||||
[[ -z "$CONTAINER_STATS" ]] && continue
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
warn "$container not found — skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
MEM_USAGE=$(echo "$CONTAINER_STATS" | cut -d'|' -f2 | awk '{print $1}')
|
||||
MEM_UNIT=$(echo "$MEM_USAGE" | grep -oE '[A-Za-z]+')
|
||||
MEM_VALUE=$(echo "$MEM_USAGE" | grep -oE '[0-9.]+')
|
||||
|
||||
case "$MEM_UNIT" in
|
||||
GiB|GB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE * 1024}") ;;
|
||||
MiB|MB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE}") ;;
|
||||
KiB|KB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE / 1024}") ;;
|
||||
*) MEM_MB=0 ;;
|
||||
esac
|
||||
|
||||
MEM_PCT=$(awk "BEGIN {printf \"%.0f\", $MEM_MB * 100 / $MEM_LIMIT_MB}")
|
||||
CPU_RAW=$(echo "$CONTAINER_STATS" | cut -d'|' -f3 | tr -d '%')
|
||||
CPU_NORM=$(awk "BEGIN {printf \"%.1f\", $CPU_RAW / $TOTAL_CORES}")
|
||||
CPU_INT=$(printf "%.0f" "$CPU_NORM")
|
||||
|
||||
# Memory check
|
||||
if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then
|
||||
error "$container — memory ${MEM_VALUE}${MEM_UNIT} exceeded hard limit ${MEM_LIMIT_MB}MB"
|
||||
safe_restart "$container" "memory hard limit exceeded"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container exceeded memory limit on $(hostname) — ${MEM_VALUE}${MEM_UNIT} / ${MEM_LIMIT_MB}MB — restarted" "warning"
|
||||
elif [[ "$MEM_PCT" -ge "$SOFT_MEM_THRESHOLD" ]]; then
|
||||
warn "$container — memory ${MEM_VALUE}${MEM_UNIT} (${MEM_PCT}% of ${MEM_LIMIT_MB}MB)"
|
||||
((T1_WARNINGS++))
|
||||
else
|
||||
success "$container — memory ${MEM_VALUE}${MEM_UNIT} (${MEM_PCT}%) CPU ${CPU_NORM}%"
|
||||
fi
|
||||
|
||||
if ! docker ps --filter "name=^/${container}$" --format "{{.Names}}" | grep -qw "$container"; then
|
||||
warn "$ICON_NOT_RUNNING $container is not running — skipping resource checks"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
# CPU check
|
||||
if [[ "$CPU_INT" -ge "$HARD_CPU_THRESHOLD" ]]; then
|
||||
CPU_STRIKES=$(get_strikes "${container}_cpu" "$WATCHDOG_STATE_FILE")
|
||||
CPU_STRIKES=$(( CPU_STRIKES + 1 ))
|
||||
set_strikes "${container}_cpu" "$CPU_STRIKES" "$WATCHDOG_STATE_FILE"
|
||||
warn "$container — CPU ${CPU_NORM}% (strike $CPU_STRIKES/$CPU_FAIL_LIMIT)"
|
||||
if [[ "$CPU_STRIKES" -ge "$CPU_FAIL_LIMIT" ]]; then
|
||||
safe_restart "$container" "CPU threshold exceeded"
|
||||
set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container CPU ${CPU_NORM}% on $(hostname) — restarted after $CPU_FAIL_LIMIT strikes" "warning"
|
||||
fi
|
||||
else
|
||||
set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
check_memory "$container" "${WATCHDOG_CONTAINERS[$container]}"
|
||||
check_cpu "$container"
|
||||
check_responsiveness "$container"
|
||||
# ── HTTP Responsiveness ──────────────────────────────────────────────────────────────────────
|
||||
if [[ ${#WATCHDOG_CONTAINER_URLS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET HTTP Responsiveness ━━━"
|
||||
|
||||
for container in "${!WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
URL="${WATCHDOG_CONTAINER_URLS[$container]}"
|
||||
if curl -sf --max-time "$CURL_TIMEOUT" "$URL" >/dev/null 2>&1; then
|
||||
success "$container — responding at $URL"
|
||||
set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
|
||||
else
|
||||
HTTP_STRIKES=$(get_strikes "${container}_http" "$WATCHDOG_STATE_FILE")
|
||||
HTTP_STRIKES=$(( HTTP_STRIKES + 1 ))
|
||||
set_strikes "${container}_http" "$HTTP_STRIKES" "$WATCHDOG_STATE_FILE"
|
||||
warn "$container — not responding at $URL (strike $HTTP_STRIKES/$RESP_FAIL_LIMIT)"
|
||||
((T1_WARNINGS++))
|
||||
if [[ "$HTTP_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then
|
||||
safe_restart "$container" "HTTP unresponsive"
|
||||
set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Required container monitoring — WATCHDOG_REQUIRED_CONTAINERS
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# TIER 2 — Global Health Scan
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Required Container Check ━━━"
|
||||
check_required_containers
|
||||
|
||||
END=$(date +%s)
|
||||
T2_RESTARTS=0
|
||||
T2_WARNINGS=0
|
||||
|
||||
if [[ "$WATCHDOG_SCAN_ALL" != "true" ]]; then
|
||||
info "Global scan disabled (WATCHDOG_SCAN_ALL=false) — skipping Tier 2"
|
||||
else
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " $ICON_WATCHDOG TIER 2 — Global Health Scan"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
ALL_CONTAINERS=$(docker ps --format "{{.Names}}" 2>/dev/null)
|
||||
|
||||
# ── Unhealthy containers ─────────────────────────────────────────────────────────────────
|
||||
if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_HEALTH Unhealthy Status ━━━"
|
||||
UNHEALTHY=$(docker ps --filter health=unhealthy --format "{{.Names}}" 2>/dev/null)
|
||||
|
||||
if [[ -z "$UNHEALTHY" ]]; then
|
||||
success "No unhealthy containers"
|
||||
else
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && \
|
||||
{ info "Skipping $container (ignore list)"; continue; }
|
||||
is_skipped "$container" && \
|
||||
{ warn "$container — unhealthy but on skip list, manual intervention needed"; continue; }
|
||||
|
||||
error "$container — Docker health status: unhealthy"
|
||||
((T2_WARNINGS++))
|
||||
result=0
|
||||
safe_restart "$container" "unhealthy health status" || result=$?
|
||||
if [[ $result -eq 0 ]]; then
|
||||
((T2_RESTARTS++))
|
||||
queue_notify "$container unhealthy on $(hostname) — restarted" "warning"
|
||||
elif [[ $result -eq 2 ]]; then
|
||||
: # skip list message already queued
|
||||
fi
|
||||
done <<< "$UNHEALTHY"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── OOM killed containers ────────────────────────────────────────────────────────────────
|
||||
if [[ "$WATCHDOG_NOTIFY_OOM" == "true" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_MEM OOM Kill Check ━━━"
|
||||
OOM_FOUND=false
|
||||
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
|
||||
OOM=$(docker inspect -f '{{.State.OOMKilled}}' "$container" 2>/dev/null)
|
||||
if [[ "$OOM" == "true" ]]; then
|
||||
error "$container — OOM killed by kernel"
|
||||
OOM_FOUND=true
|
||||
((T2_WARNINGS++))
|
||||
result=0
|
||||
safe_restart "$container" "OOM killed" || result=$?
|
||||
if [[ $result -eq 0 ]]; then
|
||||
((T2_RESTARTS++))
|
||||
queue_notify "$container OOM killed on $(hostname) — system under memory pressure — restarted" "warning"
|
||||
fi
|
||||
fi
|
||||
done <<< "$ALL_CONTAINERS"
|
||||
|
||||
[[ "$OOM_FOUND" == false ]] && success "No OOM kills detected"
|
||||
fi
|
||||
|
||||
# ── Crash loop detection ─────────────────────────────────────────────────────────────────
|
||||
if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_REBOOT_SMART Crash Loop Detection ━━━"
|
||||
CRASH_FOUND=false
|
||||
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
|
||||
RESTART_COUNT=$(docker inspect -f '{{.RestartCount}}' "$container" 2>/dev/null || echo 0)
|
||||
PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d: -f2 || echo 0)
|
||||
|
||||
# Update baseline
|
||||
set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE"
|
||||
|
||||
if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then
|
||||
CRASH_FOUND=true
|
||||
((T2_WARNINGS++))
|
||||
if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then
|
||||
error "$container — crash loop CRITICAL: $RESTART_COUNT total Docker restarts"
|
||||
queue_notify "$container crash loop CRITICAL on $(hostname) — $RESTART_COUNT Docker restarts — manual intervention needed" "critical"
|
||||
else
|
||||
warn "$container — Docker restarted since last check (total: $RESTART_COUNT)"
|
||||
queue_notify "$container restarted on $(hostname) — Docker restart count: $RESTART_COUNT" "warning"
|
||||
fi
|
||||
elif [[ "$RESTART_COUNT" -gt 0 ]]; then
|
||||
log "$container — Docker restart count: $RESTART_COUNT (stable)"
|
||||
fi
|
||||
done <<< "$ALL_CONTAINERS"
|
||||
|
||||
[[ "$CRASH_FOUND" == false ]] && success "No crash loops detected"
|
||||
fi
|
||||
|
||||
# ── Dead containers ──────────────────────────────────────────────────────────────────────
|
||||
if [[ "$WATCHDOG_RESTART_DEAD" == "true" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOPPED Dead Containers ━━━"
|
||||
DEAD=$(docker ps -a --filter status=dead --format "{{.Names}}" 2>/dev/null)
|
||||
|
||||
if [[ -z "$DEAD" ]]; then
|
||||
success "No dead containers"
|
||||
else
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && \
|
||||
{ info "Skipping $container (ignore list)"; continue; }
|
||||
is_skipped "$container" && \
|
||||
{ warn "$container — dead but on skip list, manual intervention needed"; continue; }
|
||||
|
||||
error "$container — status: dead"
|
||||
((T2_WARNINGS++))
|
||||
RESTART_COUNT=$(get_restart_count "$container")
|
||||
if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then
|
||||
add_to_skip_list "$container" "dead state — restarted $RESTART_COUNT times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
|
||||
elif [[ "$DRY_RUN" == false ]]; then
|
||||
docker rm "$container" >/dev/null 2>&1
|
||||
if docker start "$container" >/dev/null 2>&1; then
|
||||
success "$container removed from dead state and restarted"
|
||||
log_restart "$container"
|
||||
((T2_RESTARTS++))
|
||||
queue_notify "$container was dead on $(hostname) — removed and restarted" "warning"
|
||||
else
|
||||
error "$container failed to restart after dead state"
|
||||
queue_notify "$container dead and failed to restart on $(hostname) — manual intervention needed" "warning"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would remove and restart $container"
|
||||
fi
|
||||
done <<< "$DEAD"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Unexpected exits ─────────────────────────────────────────────────────────────────────
|
||||
if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_ERROR Unexpected Exits ━━━"
|
||||
CRASHED=$(docker ps -a \
|
||||
--filter status=exited \
|
||||
--format "{{.Names}}|{{.Status}}" 2>/dev/null | \
|
||||
grep -v "Exited (0)")
|
||||
|
||||
if [[ -z "$CRASHED" ]]; then
|
||||
success "No unexpected exits"
|
||||
else
|
||||
while IFS='|' read -r container status; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && \
|
||||
{ info "Skipping $container (ignore list)"; continue; }
|
||||
is_skipped "$container" && \
|
||||
{ warn "$container — crashed but on skip list, manual intervention needed"; continue; }
|
||||
|
||||
# Skip Tier 1 required containers — already handled above
|
||||
SKIP=false
|
||||
for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
|
||||
[[ "$container" == "$req" ]] && SKIP=true && break
|
||||
done
|
||||
[[ "$SKIP" == true ]] && continue
|
||||
|
||||
error "$container — $status (unexpected exit)"
|
||||
((T2_WARNINGS++))
|
||||
result=0
|
||||
safe_restart "$container" "unexpected exit" || result=$?
|
||||
if [[ $result -eq 0 ]]; then
|
||||
((T2_RESTARTS++))
|
||||
queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning"
|
||||
elif [[ $result -eq 2 ]]; then
|
||||
: # skip list message already queued
|
||||
fi
|
||||
done <<< "$CRASHED"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Send batched notifications
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
flush_notify
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
TOTAL_RESTARTS=$(( T1_RESTARTS + T2_RESTARTS ))
|
||||
TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS ))
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_WATCHDOG Monitored: ${#WATCHDOG_CONTAINERS[@]} containers"
|
||||
echo "$ICON_CONTAINERS Required: ${#WATCHDOG_REQUIRED_CONTAINERS[@]} containers"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]}"
|
||||
[[ "$DRY_RUN" == true ]] && echo "$ICON_WARN Dry Run: no actions taken"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER WATCHDOG SUMMARY ━━━━━"
|
||||
echo "$ICON_WATCHDOG Tier 1 (strict): $T1_RESTARTS restarts / $T1_WARNINGS warnings"
|
||||
echo "$ICON_WATCHDOG Tier 2 (global): $T2_RESTARTS restarts / $T2_WARNINGS warnings"
|
||||
echo "$ICON_WATCHDOG Total: $TOTAL_RESTARTS restarts / $TOTAL_WARNINGS warnings"
|
||||
[[ "$IN_GRACE_PERIOD" == "true" ]] && \
|
||||
echo "$ICON_WARN Grace period: active — restarts suppressed"
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then
|
||||
echo "$ICON_WARN Status: $TOTAL_RESTARTS container(s) restarted / $TOTAL_WARNINGS warning(s)"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL CONTAINERS HEALTHY"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
+90
-36
@@ -380,33 +380,36 @@ WEEKLY_RESTART_CONTAINERS=(
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# First line of defense for container health — runs every 15 minutes via cron.
|
||||
# Monitors memory usage, CPU usage and HTTP responsiveness per container.
|
||||
# Uses a strike system to avoid restarting on brief spikes — sustained issues trigger restart.
|
||||
# Works alongside system_watchdog.sh — containers first, system reboot is the last resort.
|
||||
|
||||
# Containers to monitor with their memory hard limits in MB.
|
||||
# Memory hard limit exceeded → immediate restart (no strike system for memory).
|
||||
# CPU and HTTP use strike system — see CPU_FAIL_LIMIT and RESP_FAIL_LIMIT below.
|
||||
# Two-tier self-healing container monitoring:
|
||||
# Tier 1 — strict monitoring of explicitly configured containers
|
||||
# Tier 2 — global health scan of ALL running containers
|
||||
#
|
||||
# Cross-cutting intelligence applies to both tiers:
|
||||
# Startup grace — skip restarts while system is still booting
|
||||
# Dependency order — restart database before app, not the other way around
|
||||
# Restart loop — stop restarting after limit hit → skip list → notify critical
|
||||
# Skip list — persistent across reboots, auto-clears when container recovers
|
||||
# Batch notify — one clean summary per run instead of one ping per event
|
||||
|
||||
# ── Tier 1 — Strict Monitoring ────────────────────────────────────────────────────────────
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded, no strike system
|
||||
# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240
|
||||
# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=16384 # 16GB — media server, transcoding can spike high
|
||||
["LidaTube"]=6144 # 6GB — YouTube downloader
|
||||
["Tdarr"]=6144 # 6GB — transcoding node
|
||||
["Code-Server"]=1024 # 1GB — VS Code server
|
||||
["Emby"]=16384
|
||||
["LidaTube"]=6144
|
||||
["Tdarr"]=6144
|
||||
["Code-Server"]=1024
|
||||
)
|
||||
|
||||
# Containers to check HTTP responsiveness via curl — omit a container to skip its HTTP check.
|
||||
# curl checks the URL and considers the container unresponsive if it times out or errors.
|
||||
|
||||
# HTTP responsiveness checks — omit container to skip its HTTP check
|
||||
declare -A WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
)
|
||||
|
||||
# Containers that should always be running — monitored for unexpected stops.
|
||||
# Strike system used — tries restart on each strike up to SYS_WATCHDOG_STRIKE_LIMIT.
|
||||
# If restart fails after strike limit → added to persistent skip list on /boot/
|
||||
# Skip list auto-clears when container recovers after reboot or manual fix.
|
||||
|
||||
# Containers that must always be running — strike system, persistent skip list on /boot/
|
||||
# Skip list auto-clears when container recovers — no manual intervention for normal recovery
|
||||
WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
@@ -416,23 +419,74 @@ WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"Authelia-Secondary"
|
||||
"Redis-Authelia-Secondary"
|
||||
)
|
||||
|
||||
# Strike state file — /tmp resets on reboot which is correct behaviour for strike tracking
|
||||
|
||||
# Strike thresholds
|
||||
WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
|
||||
|
||||
# CPU thresholds — normalised against total core count automatically at runtime.
|
||||
# A container using 85% of one core on a 16-core system = ~5.3% normalised.
|
||||
SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
|
||||
HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU
|
||||
CPU_FAIL_LIMIT=2 # consecutive strikes before container restart
|
||||
|
||||
# Memory soft threshold — warn when container reaches this % of its hard limit.
|
||||
# Hard limit exceeded triggers immediate restart regardless of strikes.
|
||||
SOFT_MEM_THRESHOLD=80
|
||||
|
||||
# HTTP responsiveness check settings
|
||||
RESP_FAIL_LIMIT=2 # consecutive failed curl checks before restart
|
||||
CURL_TIMEOUT=5 # seconds before curl gives up per check
|
||||
SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
|
||||
HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU
|
||||
CPU_FAIL_LIMIT=2 # consecutive CPU strikes before restart
|
||||
SOFT_MEM_THRESHOLD=80 # warn when container reaches this % of hard limit
|
||||
RESP_FAIL_LIMIT=2 # consecutive failed HTTP checks before restart
|
||||
CURL_TIMEOUT=5 # seconds before curl gives up per check
|
||||
|
||||
# ── Tier 2 — Global Health Scan ───────────────────────────────────────────────────────────
|
||||
|
||||
# Master toggle — false disables Tier 2 entirely
|
||||
WATCHDOG_SCAN_ALL=true
|
||||
|
||||
# Containers to skip in Tier 2 — add containers expected to be in a non-running state
|
||||
# or managed by other systems that should not be auto-restarted
|
||||
WATCHDOG_SCAN_IGNORE=(
|
||||
# "container-name"
|
||||
)
|
||||
|
||||
# Individual Tier 2 check toggles — disable checks that cause false positives
|
||||
WATCHDOG_RESTART_UNHEALTHY=true # restart containers with unhealthy Docker health status
|
||||
WATCHDOG_RESTART_DEAD=true # remove and restart containers in dead state
|
||||
WATCHDOG_RESTART_CRASHED=true # restart containers that exited with non-zero exit code
|
||||
WATCHDOG_NOTIFY_OOM=true # restart and notify when OOM killed by kernel
|
||||
WATCHDOG_NOTIFY_CRASHLOOP=true # notify when Docker restart count is climbing
|
||||
|
||||
# Crash loop threshold — notify critical if Docker has restarted container this many times
|
||||
WATCHDOG_CRASH_LIMIT=5
|
||||
|
||||
# ── Cross-cutting Intelligence ────────────────────────────────────────────────────────────
|
||||
|
||||
# Startup grace period — skip restarts while system is still booting
|
||||
# Prevents false positives while containers are coming up after array start
|
||||
WATCHDOG_STARTUP_GRACE=300 # seconds after boot before watchdog acts on failures
|
||||
|
||||
# Restart loop protection — stops hammering broken containers
|
||||
# Tracks watchdog-initiated restarts per container in a bounded /boot/ file
|
||||
# After limit hit → container added to skip list → notify critical → manual intervention
|
||||
# Skip list auto-clears when container is found running again
|
||||
WATCHDOG_CONTAINER_RESTART_LIMIT=3 # max watchdog restarts allowed in window
|
||||
WATCHDOG_CONTAINER_RESTART_WINDOW=1 # hours — rolling window for restart count
|
||||
WATCHDOG_CONTAINER_RESTART_LOG="/boot/config/container_restart_history.db"
|
||||
# /boot/ survives reboots — bounded, auto-purges old entries
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down
|
||||
# Dependency gets restarted first, dependent picked up on the next watchdog cycle
|
||||
# Format: ["dependent"]="dependency1 dependency2"
|
||||
declare -A WATCHDOG_DEPENDENCIES=(
|
||||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||||
["NextCloud"]="Postgres-NextCloud"
|
||||
)
|
||||
|
||||
# Notification batching — one clean summary per run instead of one ping per event
|
||||
# true = batch all events into a single notification at end of run
|
||||
# false = send individual notification per event as it happens
|
||||
WATCHDOG_BATCH_NOTIFY=true
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
NETWORK_CONNECT_CONTAINERS=(
|
||||
"memcached"
|
||||
"Npm-CrowdSec"
|
||||
)
|
||||
NETWORK_CONNECT_NETWORKS=(
|
||||
"nextcloud-aio"
|
||||
)
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
# Connects containers to extra Docker networks on array start.
|
||||
|
||||
Reference in New Issue
Block a user