added more descriptions to master.conf
This commit is contained in:
@@ -1,40 +1,44 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- System Watchdog --------------------------------------------
|
||||
# --------------------------------- Docker Watchdog --------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Last line of defense — reboots the system cleanly if it is about to become unstable.
|
||||
# Runs continuously as a background process — started by array_start.sh at array start.
|
||||
# Works alongside docker_watchdog.sh which handles container-level healing first.
|
||||
# Two-tier self-healing container monitoring system — runs continuously as a background process.
|
||||
# Started by array_start.sh at array start — runs until array stops or SIGTERM received.
|
||||
#
|
||||
# Checks (all toggleable in Master.conf):
|
||||
# rootfs usage — high rootfs fills rapidly when array is down, crash imminent
|
||||
# /var/log usage — log spam can fill rootfs, indicates something is broken
|
||||
# free RAM — critically low RAM means OOM or swap imminent
|
||||
# ZFS ARC pinned — ARC not releasing after reclaim means memory is stuck
|
||||
# CPU temperature — sustained tjmax causes throttling or kernel panic
|
||||
# load average — sustained high load means something is stuck or runaway
|
||||
# zombie processes — large zombie count indicates serious process management failure
|
||||
# Docker daemon — unresponsive daemon means containers cannot be managed
|
||||
# Required containers — stopped containers that should be running (after watchdog skip list)
|
||||
# Tier 1 — Strict monitoring (configured containers only)
|
||||
# Memory hard limits — immediate restart if exceeded
|
||||
# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT strikes
|
||||
# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT strikes
|
||||
# Required containers — must always be running, strike system with skip list
|
||||
#
|
||||
# Abort conditions (toggleable):
|
||||
# ZFS pool unhealthy — reboot with bad pool risks data loss
|
||||
# Parity running — aborting parity is better than crashing mid-check
|
||||
# Mover running — aborting move is better than crashing mid-move
|
||||
# Tier 2 — Global health scan (all running containers)
|
||||
# Unhealthy status — Docker HEALTHCHECK unhealthy → restart
|
||||
# OOM killed — kernel killed container → restart + notify
|
||||
# Crash loop detection — RestartCount climbing → notify, critical above limit
|
||||
# Dead containers — remove and restart
|
||||
# Unexpected exits — non-zero exit code → restart
|
||||
#
|
||||
# Reboot loop protection:
|
||||
# Tracks reboot timestamps in persistent log on /boot/
|
||||
# Rolling window — old entries purge automatically
|
||||
# If reboot count hits limit in window → shutdown instead of reboot
|
||||
# Cross-cutting intelligence:
|
||||
# Startup grace period — skip restarts while system is still booting
|
||||
# Dependency ordering — restart database before app
|
||||
# Restart loop protect — stop restarting after X restarts in X hours → skip list
|
||||
# Skip list auto-clear — clears when container recovers
|
||||
# Notification batching — one clean summary per cycle, not one ping per event
|
||||
# Quiet when healthy — only logs when something needs attention
|
||||
# Parity awareness — skips restarts during parity check
|
||||
#
|
||||
# Continuous loop:
|
||||
# Checks run every SYSTEM_WATCHDOG_INTERVAL seconds (default 300 = 5min)
|
||||
# Master.conf re-sourced each cycle — config changes picked up without restart
|
||||
# Silent when healthy — only verbose when trigger or reboot
|
||||
# Checks run every DOCKER_WATCHDOG_INTERVAL seconds (default 900 = 15min)
|
||||
# Clean shutdown on SIGTERM/SIGINT — sent by array stop
|
||||
# Variables scoped per-cycle — no state accumulation between cycles
|
||||
#
|
||||
# All configuration in Master.conf under System Watchdog section.
|
||||
# Supports --dry-run to show triggered conditions without rebooting.
|
||||
# State files:
|
||||
# WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot)
|
||||
# SYS_WATCHDOG_FAILED_FILE — persistent skip list (/boot — survives reboots)
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection
|
||||
#
|
||||
# All configuration in Master.conf under Docker Watchdog section.
|
||||
# Supports --dry-run and --status.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -59,23 +63,37 @@ success "Running as root"
|
||||
|
||||
acquire_lock "continuous"
|
||||
|
||||
TOTAL_CORES=$(nproc)
|
||||
# Select correct per-host watchdog lists — done once at startup
|
||||
detect_hosts
|
||||
|
||||
# Ensure state and persistent files exist
|
||||
touch "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || {
|
||||
error "Cannot create state file: $SYS_WATCHDOG_STATE_FILE"
|
||||
exit 1
|
||||
}
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
declare -A WATCHDOG_CONTAINER_URLS
|
||||
for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}"
|
||||
done
|
||||
else
|
||||
WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
declare -A WATCHDOG_CONTAINER_URLS
|
||||
for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}"
|
||||
done
|
||||
fi
|
||||
|
||||
touch "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || {
|
||||
error "Cannot create reboot log: $SYS_WATCHDOG_REBOOT_LOG"
|
||||
exit 1
|
||||
}
|
||||
info "Watchdog running as: $LOCAL_SERVER_NAME"
|
||||
info "Check interval: ${DOCKER_WATCHDOG_INTERVAL}s"
|
||||
|
||||
touch "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || {
|
||||
error "Cannot create failed container list: $SYS_WATCHDOG_FAILED_FILE"
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
error "Docker not found"
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
|
||||
success "Docker found"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||
|
||||
# Ensure state files exist
|
||||
touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \
|
||||
"$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
@@ -83,198 +101,158 @@ touch "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || {
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HEALTH rootfs threshold: ${SYS_WATCHDOG_ROOTFS_PCT}% toggle: $SYS_WATCHDOG_CHECK_ROOTFS"
|
||||
echo "$ICON_HEALTH log threshold: ${SYS_WATCHDOG_LOG_PCT}% toggle: $SYS_WATCHDOG_CHECK_LOG"
|
||||
echo "$ICON_MEM RAM threshold: ${SYS_WATCHDOG_MEM_GB}GB free toggle: $SYS_WATCHDOG_CHECK_RAM"
|
||||
echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}% toggle: $SYS_WATCHDOG_CHECK_ARC"
|
||||
echo "$ICON_ZFS ARC release: ${SYS_WATCHDOG_ARC_RELEASE_PCT}%"
|
||||
echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C toggle: $SYS_WATCHDOG_CHECK_CPU_TEMP"
|
||||
echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x cores toggle: $SYS_WATCHDOG_CHECK_LOAD"
|
||||
echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT} toggle: $SYS_WATCHDOG_CHECK_ZOMBIES"
|
||||
echo "$ICON_CONTAINERS Docker daemon: toggle: $SYS_WATCHDOG_CHECK_DOCKER_DAEMON"
|
||||
echo "$ICON_CONTAINERS Containers: toggle: $SYS_WATCHDOG_CHECK_CONTAINERS"
|
||||
echo "$ICON_SHIELD Strike limit: $SYS_WATCHDOG_STRIKE_LIMIT"
|
||||
echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s"
|
||||
echo "$ICON_REBOOT_SMART Reboot limit: $SYS_WATCHDOG_REBOOT_LIMIT in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs"
|
||||
echo "$ICON_ZFS Abort ZFS unhealthy: $SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY"
|
||||
echo "$ICON_GEAR Abort on parity: $SYS_WATCHDOG_ABORT_ON_PARITY"
|
||||
echo "$ICON_MOVER Abort on mover: $SYS_WATCHDOG_ABORT_ON_MOVER"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_CONTAINERS Watched: ${!WATCHDOG_CONTAINERS[@]}"
|
||||
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}"
|
||||
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
|
||||
echo "$ICON_WATCHDOG Interval: ${DOCKER_WATCHDOG_INTERVAL}s"
|
||||
echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s"
|
||||
echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
|
||||
echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY"
|
||||
echo "$ICON_WATCHDOG Ignore list: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
|
||||
echo "$ICON_TIME System uptime: $(format_duration $UPTIME_S)"
|
||||
[[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]] && \
|
||||
warn "Within startup grace period — restarts suppressed"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboot will be executed"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# STATE HELPERS — defined once, used every cycle
|
||||
# HELPERS — defined once, used every cycle
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
get_strikes() {
|
||||
local key="$1"
|
||||
grep -E "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2
|
||||
grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"
|
||||
}
|
||||
|
||||
set_strikes() {
|
||||
local key="$1" count="$2"
|
||||
grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
||||
echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
||||
mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE"
|
||||
local container="$1" count="$2" file="$3"
|
||||
if grep -q "^${container}:" "$file" 2>/dev/null; then
|
||||
sed -i "s/^${container}:.*/${container}:${count}/" "$file"
|
||||
else
|
||||
echo "${container}:${count}" >> "$file"
|
||||
fi
|
||||
}
|
||||
|
||||
increment_strikes() {
|
||||
local key="$1"
|
||||
local current
|
||||
current=$(get_strikes "$key")
|
||||
[[ -z "$current" ]] && current=0
|
||||
((current++))
|
||||
set_strikes "$key" "$current"
|
||||
echo "$current"
|
||||
is_skipped() {
|
||||
grep -q "^${1}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
}
|
||||
|
||||
reset_strikes() {
|
||||
local key="$1"
|
||||
set_strikes "$key" 0
|
||||
add_to_skip_list() {
|
||||
local container="$1" reason="$2"
|
||||
if ! is_skipped "$container"; then
|
||||
echo "$container" >> "$SYS_WATCHDOG_FAILED_FILE"
|
||||
error "$container added to skip list — $reason"
|
||||
queue_notify "$container added to skip list on $(hostname) — $reason — manual intervention needed" "critical"
|
||||
fi
|
||||
}
|
||||
|
||||
purge_old_reboots() {
|
||||
remove_from_skip_list() {
|
||||
sed -i "/^${1}$/d" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
success "$1 removed from skip list — recovered"
|
||||
}
|
||||
|
||||
log_restart() {
|
||||
local container="$1"
|
||||
local now
|
||||
now=$(date +%s)
|
||||
local cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW ))
|
||||
grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do
|
||||
[[ "$ts" -gt "$cutoff" ]] && echo "$ts"
|
||||
done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp"
|
||||
mv "${SYS_WATCHDOG_REBOOT_LOG}.tmp" "$SYS_WATCHDOG_REBOOT_LOG"
|
||||
now=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
local cutoff
|
||||
cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S')
|
||||
echo "${container}|${now}" >> "$WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
local tmp="${WATCHDOG_CONTAINER_RESTART_LOG}.tmp"
|
||||
awk -F'|' -v cutoff="$cutoff" '$2 >= cutoff' \
|
||||
"$WATCHDOG_CONTAINER_RESTART_LOG" > "$tmp" && \
|
||||
mv "$tmp" "$WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
}
|
||||
|
||||
count_recent_reboots() {
|
||||
purge_old_reboots
|
||||
grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0
|
||||
get_restart_count() {
|
||||
local container="$1"
|
||||
local cutoff
|
||||
cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S')
|
||||
awk -F'|' -v c="$container" -v cutoff="$cutoff" \
|
||||
'$1==c && $2>=cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l
|
||||
}
|
||||
|
||||
log_reboot() {
|
||||
date +%s >> "$SYS_WATCHDOG_REBOOT_LOG"
|
||||
}
|
||||
|
||||
check_abort_conditions() {
|
||||
local should_abort=false
|
||||
|
||||
if command -v zpool >/dev/null 2>&1; then
|
||||
local unhealthy
|
||||
unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true)
|
||||
if [[ -n "$unhealthy" ]]; then
|
||||
if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then
|
||||
error "$ICON_ZFS ZFS pool unhealthy — aborting reboot"
|
||||
notify "System watchdog aborted reboot on $(hostname) — ZFS pool unhealthy" "System Watchdog" "warning"
|
||||
should_abort=true
|
||||
else
|
||||
warn "$ICON_ZFS ZFS pool unhealthy — continuing reboot"
|
||||
fi
|
||||
dependencies_satisfied() {
|
||||
local container="$1"
|
||||
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
|
||||
[[ -z "$deps" ]] && return 0
|
||||
for dep in $deps; do
|
||||
local status
|
||||
status=$(docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null)
|
||||
if [[ "$status" != "true" ]]; then
|
||||
warn "$container — dependency $dep is not running — skipping restart this cycle"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -f /var/local/emhttp/parity-date.txt ]]; then
|
||||
if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then
|
||||
if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then
|
||||
error "$ICON_DISK Parity check running — aborting reboot"
|
||||
notify "System watchdog aborted reboot on $(hostname) — parity running" "System Watchdog" "warning"
|
||||
should_abort=true
|
||||
else
|
||||
warn "$ICON_DISK Parity check running — continuing reboot"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if pgrep -f "mover" >/dev/null 2>&1; then
|
||||
if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then
|
||||
error "$ICON_MOVER Mover running — aborting reboot"
|
||||
notify "System watchdog aborted reboot on $(hostname) — mover running" "System Watchdog" "warning"
|
||||
should_abort=true
|
||||
else
|
||||
warn "$ICON_MOVER Mover running — continuing reboot"
|
||||
fi
|
||||
fi
|
||||
|
||||
[[ "$should_abort" == true ]] && return 1
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
run_strike_check() {
|
||||
local key="$1" triggered="$2" description="$3"
|
||||
if [[ "$triggered" == true ]]; then
|
||||
local strikes
|
||||
strikes=$(increment_strikes "$key")
|
||||
warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT"
|
||||
if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then
|
||||
error "$description hit strike limit — reboot triggered"
|
||||
reset_strikes "$key"
|
||||
return 0
|
||||
fi
|
||||
safe_restart() {
|
||||
local container="$1" reason="$2"
|
||||
local restart_count
|
||||
restart_count=$(get_restart_count "$container")
|
||||
if [[ "$restart_count" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then
|
||||
add_to_skip_list "$container" \
|
||||
"restarted $restart_count times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
|
||||
return 2
|
||||
fi
|
||||
dependencies_satisfied "$container" || return 1
|
||||
local uptime_s
|
||||
uptime_s=$(awk '{print int($1)}' /proc/uptime)
|
||||
if [[ "$uptime_s" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||
warn "$container — within startup grace period — skipping restart"
|
||||
return 1
|
||||
fi
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $container ($reason)"
|
||||
return 0
|
||||
fi
|
||||
info "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]..."
|
||||
if docker restart "$container" >/dev/null 2>&1; then
|
||||
success "$ICON_STARTED $container restarted"
|
||||
log_restart "$container"
|
||||
return 0
|
||||
else
|
||||
local current
|
||||
current=$(get_strikes "$key")
|
||||
if [[ -n "$current" && "$current" -gt 0 ]]; then
|
||||
reset_strikes "$key"
|
||||
fi
|
||||
error "Failed to restart $container"
|
||||
return 1
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
do_reboot() {
|
||||
local triggers=("$@")
|
||||
queue_notify() {
|
||||
local message="$1" severity="${2:-warning}"
|
||||
NOTIFY_EVENTS+=("${severity}|${message}")
|
||||
log "Queued: $message"
|
||||
}
|
||||
|
||||
if ! check_abort_conditions; then
|
||||
return
|
||||
fi
|
||||
|
||||
RECENT_REBOOTS=$(count_recent_reboots)
|
||||
info "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT"
|
||||
|
||||
if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then
|
||||
error "Reboot limit hit — shutting down instead"
|
||||
notify "Reboot loop detected on $(hostname) — shutting down after $RECENT_REBOOTS reboots — conditions: ${triggers[*]}" "System Watchdog" "warning"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would shutdown now"
|
||||
return
|
||||
fi
|
||||
sync
|
||||
/sbin/poweroff
|
||||
return
|
||||
fi
|
||||
|
||||
notify "System watchdog reboot triggered on $(hostname) — conditions: ${triggers[*]}" "System Watchdog" "warning"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — reboot sequence would begin now"
|
||||
return
|
||||
fi
|
||||
|
||||
log_reboot
|
||||
|
||||
info "Shutting down VMs..."
|
||||
if command -v virsh >/dev/null 2>&1; then
|
||||
for VM in $(virsh list --name 2>/dev/null); do
|
||||
[[ -z "$VM" ]] && continue
|
||||
virsh shutdown "$VM" >/dev/null 2>&1
|
||||
flush_notify() {
|
||||
[[ ${#NOTIFY_EVENTS[@]} -eq 0 ]] && return
|
||||
if [[ "$WATCHDOG_BATCH_NOTIFY" == "true" ]]; then
|
||||
local highest_severity="normal"
|
||||
local messages=()
|
||||
for event in "${NOTIFY_EVENTS[@]}"; do
|
||||
local sev="${event%%|*}" msg="${event#*|}"
|
||||
messages+=("$msg")
|
||||
[[ "$sev" == "critical" ]] && highest_severity="warning"
|
||||
[[ "$sev" == "warning" && "$highest_severity" == "normal" ]] && highest_severity="warning"
|
||||
done
|
||||
local summary
|
||||
summary=$(printf '%s. ' "${messages[@]}")
|
||||
notify "Docker Watchdog on $(hostname) — ${#NOTIFY_EVENTS[@]} event(s): $summary" \
|
||||
"Docker Watchdog" "$highest_severity"
|
||||
else
|
||||
for event in "${NOTIFY_EVENTS[@]}"; do
|
||||
local sev="${event%%|*}" msg="${event#*|}"
|
||||
[[ "$sev" == "critical" ]] && sev="warning"
|
||||
notify "$msg" "Docker Watchdog" "$sev"
|
||||
done
|
||||
sleep 30
|
||||
fi
|
||||
NOTIFY_EVENTS=()
|
||||
}
|
||||
|
||||
info "Stopping Docker containers..."
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
docker ps -q | xargs -r docker stop >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
info "Stopping User Scripts..."
|
||||
pkill -f "/tmp/user.scripts" 2>/dev/null || true
|
||||
|
||||
info "Syncing disks..."
|
||||
sync
|
||||
|
||||
echo ""
|
||||
echo "$ICON_REBOOT_SMART Rebooting system NOW..."
|
||||
sleep 5
|
||||
/sbin/reboot
|
||||
is_parity_running() {
|
||||
grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
@@ -284,7 +262,7 @@ WATCHDOG_RUNNING=true
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
info "System watchdog received shutdown signal — stopping cleanly"
|
||||
info "Docker watchdog received shutdown signal — stopping cleanly"
|
||||
WATCHDOG_RUNNING=false
|
||||
exit 0
|
||||
}
|
||||
@@ -294,140 +272,312 @@ trap cleanup SIGTERM SIGINT
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ CONTINUOUS MONITORING LOOP ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
info "System watchdog started — checking every ${SYSTEM_WATCHDOG_INTERVAL}s"
|
||||
info "Docker watchdog started — checking every ${DOCKER_WATCHDOG_INTERVAL}s"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
CYCLE=0
|
||||
|
||||
while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
((CYCLE++))
|
||||
CYCLE_START=$(date +%s)
|
||||
|
||||
# Re-source Master.conf each cycle — picks up config changes without restart
|
||||
# Re-source Master.conf each cycle — picks up any config changes without restart
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 ))
|
||||
TOTAL_CORES=$(nproc)
|
||||
|
||||
# Per-cycle triggers — cleared each iteration
|
||||
TRIGGERS=()
|
||||
|
||||
# ── rootfs usage ─────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then
|
||||
ROOTFS_USED=$(df / --output=pcent | tail -1 | tr -d ' %')
|
||||
TRIGGERED=false
|
||||
[[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true
|
||||
run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \
|
||||
TRIGGERS+=("rootfs=${ROOTFS_USED}%")
|
||||
fi
|
||||
|
||||
# ── /var/log usage ────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then
|
||||
LOG_USED=$(df -P /var/log | awk 'NR==2 {print $5}' | tr -d '%')
|
||||
TRIGGERED=false
|
||||
[[ "$LOG_USED" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true
|
||||
run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \
|
||||
TRIGGERS+=("log=${LOG_USED}%")
|
||||
fi
|
||||
|
||||
# ── Free RAM ─────────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then
|
||||
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_GB=$((MEM_KB / 1024 / 1024))
|
||||
TRIGGERED=false
|
||||
[[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]] && TRIGGERED=true
|
||||
run_strike_check "ram" "$TRIGGERED" "RAM ${MEM_GB}GB free" && \
|
||||
TRIGGERS+=("low_ram=${MEM_GB}GB")
|
||||
fi
|
||||
|
||||
# ── ZFS ARC ──────────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||||
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX ))
|
||||
TRIGGERED=false
|
||||
if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then
|
||||
sync; echo 3 > /proc/sys/vm/drop_caches; sleep 5
|
||||
ARC_AFTER=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX ))
|
||||
[[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]] && TRIGGERED=true
|
||||
fi
|
||||
run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned" && \
|
||||
TRIGGERS+=("arc_pinned=${ARC_PCT}%")
|
||||
fi
|
||||
|
||||
# ── CPU temperature ──────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then
|
||||
CPU_TEMP=""
|
||||
if command -v sensors >/dev/null 2>&1; then
|
||||
CPU_TEMP=$(sensors 2>/dev/null | grep -i "Package id 0\|Tctl\|CPU Temp" | \
|
||||
awk '{print $NF}' | tr -d '+°C' | head -1)
|
||||
fi
|
||||
if [[ -n "$CPU_TEMP" ]]; then
|
||||
CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP")
|
||||
TRIGGERED=false
|
||||
[[ "$CPU_TEMP_INT" -ge "$SYS_WATCHDOG_CPU_TEMP_MAX" ]] && TRIGGERED=true
|
||||
run_strike_check "cpu_temp" "$TRIGGERED" "CPU temp ${CPU_TEMP_INT}°C" && \
|
||||
TRIGGERS+=("cpu_temp=${CPU_TEMP_INT}C")
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Load average ─────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_LOAD" == true ]]; then
|
||||
LOAD=$(awk '{print $1}' /proc/loadavg)
|
||||
LOAD_INT=$(printf "%.0f" "$LOAD")
|
||||
LOAD_THRESHOLD=$(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER ))
|
||||
TRIGGERED=false
|
||||
[[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true
|
||||
run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \
|
||||
TRIGGERS+=("load=${LOAD}")
|
||||
fi
|
||||
|
||||
# ── Zombie processes ─────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then
|
||||
ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" || echo 0)
|
||||
TRIGGERED=false
|
||||
[[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true
|
||||
run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \
|
||||
TRIGGERS+=("zombies=${ZOMBIE_COUNT}")
|
||||
fi
|
||||
|
||||
# ── Docker daemon ────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then
|
||||
TRIGGERED=false
|
||||
! timeout 10 docker ps >/dev/null 2>&1 && TRIGGERED=true
|
||||
run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon unresponsive" && \
|
||||
TRIGGERS+=("docker_daemon")
|
||||
fi
|
||||
|
||||
# ── Required containers from skip list ───────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
FAILED_CONTAINERS=()
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
[[ "$STATUS" != "true" ]] && FAILED_CONTAINERS+=("$container")
|
||||
done < "$SYS_WATCHDOG_FAILED_FILE"
|
||||
|
||||
if [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]]; then
|
||||
run_strike_check "failed_containers" "true" "required containers stopped" && \
|
||||
TRIGGERS+=("containers=${FAILED_CONTAINERS[*]}")
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Evaluate triggers ────────────────────────────────────────────────────────────────────
|
||||
if [[ ${#TRIGGERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_REBOOT_SMART System Watchdog — Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
for t in "${TRIGGERS[@]}"; do
|
||||
echo " $ICON_REBOOT_SMART $t"
|
||||
# Rebuild per-host lists after re-source in case they changed
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}"
|
||||
done
|
||||
echo ""
|
||||
do_reboot "${TRIGGERS[@]}"
|
||||
else
|
||||
log "Cycle $CYCLE — system healthy ($(date '+%H:%M:%S'))"
|
||||
WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}"
|
||||
done
|
||||
fi
|
||||
|
||||
# Per-cycle variables — cleared each iteration, no accumulation
|
||||
NOTIFY_EVENTS=()
|
||||
T1_RESTARTS=0
|
||||
T1_WARNINGS=0
|
||||
T2_RESTARTS=0
|
||||
T2_WARNINGS=0
|
||||
|
||||
# Rebuild ignore map each cycle in case config was updated
|
||||
declare -A IGNORE_MAP
|
||||
for c in "${WATCHDOG_SCAN_IGNORE[@]}"; do
|
||||
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
|
||||
done
|
||||
|
||||
# Startup grace check
|
||||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||||
IN_GRACE_PERIOD=false
|
||||
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||
IN_GRACE_PERIOD=true
|
||||
fi
|
||||
|
||||
# Parity check — skip restarts during parity to avoid I/O interference
|
||||
if is_parity_running; then
|
||||
log "Parity check in progress — skipping restart actions this cycle"
|
||||
sleep "$DOCKER_WATCHDOG_INTERVAL"
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── TIER 1 — Strict Monitoring ──────────────────────────────────────────────────────────
|
||||
|
||||
# Required containers
|
||||
if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
|
||||
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
|
||||
if is_skipped "$container"; then
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
remove_from_skip_list "$container"
|
||||
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
sed -i "/^${container}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
|
||||
queue_notify "$container recovered on $(hostname) — removed from skip list" "normal"
|
||||
else
|
||||
warn "$container — on skip list, manual intervention needed"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
else
|
||||
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
|
||||
STRIKES=$(( STRIKES + 1 ))
|
||||
set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
|
||||
warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
|
||||
((T1_WARNINGS++))
|
||||
|
||||
if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
|
||||
result=0
|
||||
safe_restart "$container" "required container down" || result=$?
|
||||
case $result in
|
||||
0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container was down and restarted on $(hostname)" "warning" ;;
|
||||
2) : ;;
|
||||
*) queue_notify "$container failed to restart on $(hostname)" "warning" ;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Memory and CPU monitoring
|
||||
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
|
||||
STATS=$(docker stats --no-stream \
|
||||
--format "{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}" 2>/dev/null)
|
||||
TOTAL_CORES=$(nproc 2>/dev/null || echo 1)
|
||||
|
||||
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
|
||||
MEM_LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
|
||||
CONTAINER_STATS=$(echo "$STATS" | grep "^${container}|" | head -1)
|
||||
[[ -z "$CONTAINER_STATS" ]] && continue
|
||||
|
||||
MEM_USAGE=$(echo "$CONTAINER_STATS" | cut -d'|' -f2 | awk '{print $1}')
|
||||
MEM_UNIT=$(echo "$MEM_USAGE" | grep -oE '[A-Za-z]+')
|
||||
MEM_VALUE=$(echo "$MEM_USAGE" | grep -oE '[0-9.]+')
|
||||
|
||||
case "$MEM_UNIT" in
|
||||
GiB|GB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE * 1024}") ;;
|
||||
MiB|MB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE}") ;;
|
||||
KiB|KB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE / 1024}") ;;
|
||||
*) MEM_MB=0 ;;
|
||||
esac
|
||||
|
||||
CPU_RAW=$(echo "$CONTAINER_STATS" | cut -d'|' -f3 | tr -d '%')
|
||||
CPU_NORM=$(awk "BEGIN {printf \"%.1f\", $CPU_RAW / $TOTAL_CORES}")
|
||||
CPU_INT=$(printf "%.0f" "$CPU_NORM")
|
||||
|
||||
if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then
|
||||
error "$container — memory exceeded hard limit ${MEM_LIMIT_MB}MB"
|
||||
safe_restart "$container" "memory hard limit exceeded"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container exceeded memory limit on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
|
||||
if [[ "$CPU_INT" -ge "$HARD_CPU_THRESHOLD" ]]; then
|
||||
CPU_STRIKES=$(get_strikes "${container}_cpu" "$WATCHDOG_STATE_FILE")
|
||||
CPU_STRIKES=$(( CPU_STRIKES + 1 ))
|
||||
set_strikes "${container}_cpu" "$CPU_STRIKES" "$WATCHDOG_STATE_FILE"
|
||||
if [[ "$CPU_STRIKES" -ge "$CPU_FAIL_LIMIT" ]]; then
|
||||
safe_restart "$container" "CPU threshold exceeded"
|
||||
set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container CPU ${CPU_NORM}% on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
else
|
||||
set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# HTTP responsiveness
|
||||
if [[ ${#WATCHDOG_CONTAINER_URLS[@]} -gt 0 ]]; then
|
||||
for container in "${!WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
URL="${WATCHDOG_CONTAINER_URLS[$container]}"
|
||||
if curl -sf --max-time "$CURL_TIMEOUT" "$URL" >/dev/null 2>&1; then
|
||||
set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
|
||||
else
|
||||
HTTP_STRIKES=$(get_strikes "${container}_http" "$WATCHDOG_STATE_FILE")
|
||||
HTTP_STRIKES=$(( HTTP_STRIKES + 1 ))
|
||||
set_strikes "${container}_http" "$HTTP_STRIKES" "$WATCHDOG_STATE_FILE"
|
||||
warn "$container — not responding at $URL (strike $HTTP_STRIKES/$RESP_FAIL_LIMIT)"
|
||||
((T1_WARNINGS++))
|
||||
if [[ "$HTTP_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then
|
||||
safe_restart "$container" "HTTP unresponsive"
|
||||
set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# ── TIER 2 — Global Health Scan ─────────────────────────────────────────────────────────
|
||||
if [[ "$WATCHDOG_SCAN_ALL" == "true" ]]; then
|
||||
ALL_CONTAINERS=$(docker ps --format "{{.Names}}" 2>/dev/null)
|
||||
|
||||
# Unhealthy containers
|
||||
if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then
|
||||
UNHEALTHY=$(docker ps --filter health=unhealthy --format "{{.Names}}" 2>/dev/null)
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
error "$container — unhealthy"
|
||||
((T2_WARNINGS++))
|
||||
result=0
|
||||
safe_restart "$container" "unhealthy health status" || result=$?
|
||||
[[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \
|
||||
queue_notify "$container unhealthy on $(hostname) — restarted" "warning"
|
||||
done <<< "$UNHEALTHY"
|
||||
fi
|
||||
|
||||
# OOM killed
|
||||
if [[ "$WATCHDOG_NOTIFY_OOM" == "true" ]]; then
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
OOM=$(docker inspect -f '{{.State.OOMKilled}}' "$container" 2>/dev/null)
|
||||
if [[ "$OOM" == "true" ]]; then
|
||||
error "$container — OOM killed"
|
||||
((T2_WARNINGS++))
|
||||
result=0
|
||||
safe_restart "$container" "OOM killed" || result=$?
|
||||
[[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \
|
||||
queue_notify "$container OOM killed on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
done <<< "$ALL_CONTAINERS"
|
||||
fi
|
||||
|
||||
# Crash loop detection
|
||||
if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
RESTART_COUNT=$(docker inspect -f '{{.RestartCount}}' "$container" 2>/dev/null || echo 0)
|
||||
PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d: -f2 || echo 0)
|
||||
set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE"
|
||||
if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then
|
||||
((T2_WARNINGS++))
|
||||
if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then
|
||||
error "$container — crash loop CRITICAL: $RESTART_COUNT restarts"
|
||||
queue_notify "$container crash loop CRITICAL on $(hostname) — manual intervention needed" "critical"
|
||||
else
|
||||
warn "$container — restarted since last check (total: $RESTART_COUNT)"
|
||||
queue_notify "$container restarted on $(hostname) — count: $RESTART_COUNT" "warning"
|
||||
fi
|
||||
fi
|
||||
done <<< "$ALL_CONTAINERS"
|
||||
fi
|
||||
|
||||
# Dead containers
|
||||
if [[ "$WATCHDOG_RESTART_DEAD" == "true" ]]; then
|
||||
DEAD=$(docker ps -a --filter status=dead --format "{{.Names}}" 2>/dev/null)
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
error "$container — dead"
|
||||
((T2_WARNINGS++))
|
||||
RESTART_COUNT=$(get_restart_count "$container")
|
||||
if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then
|
||||
add_to_skip_list "$container" "dead — restarted $RESTART_COUNT times"
|
||||
elif [[ "$DRY_RUN" == false ]]; then
|
||||
docker rm "$container" >/dev/null 2>&1
|
||||
if docker start "$container" >/dev/null 2>&1; then
|
||||
success "$container removed from dead state and restarted"
|
||||
log_restart "$container"
|
||||
((T2_RESTARTS++))
|
||||
queue_notify "$container was dead on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
fi
|
||||
done <<< "$DEAD"
|
||||
fi
|
||||
|
||||
# Unexpected exits
|
||||
if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then
|
||||
CRASHED=$(docker ps -a \
|
||||
--filter status=exited \
|
||||
--format "{{.Names}}|{{.Status}}" 2>/dev/null | \
|
||||
grep -v "Exited (0)")
|
||||
while IFS='|' read -r container status; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
SKIP=false
|
||||
for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
|
||||
[[ "$container" == "$req" ]] && SKIP=true && break
|
||||
done
|
||||
[[ "$SKIP" == true ]] && continue
|
||||
error "$container — $status (unexpected exit)"
|
||||
((T2_WARNINGS++))
|
||||
result=0
|
||||
safe_restart "$container" "unexpected exit" || result=$?
|
||||
[[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \
|
||||
queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning"
|
||||
done <<< "$CRASHED"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Send notifications if any events this cycle
|
||||
flush_notify
|
||||
|
||||
# Only log summary if something happened — quiet when all healthy
|
||||
TOTAL_RESTARTS=$(( T1_RESTARTS + T2_RESTARTS ))
|
||||
TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS ))
|
||||
|
||||
if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then
|
||||
CYCLE_END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━ $ICON_WATCHDOG Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings"
|
||||
echo "$ICON_WATCHDOG T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( CYCLE_END - CYCLE_START )))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
else
|
||||
log "Cycle $CYCLE — all healthy ($(date '+%H:%M:%S'))"
|
||||
# Heartbeat — periodic proof of life
|
||||
if [[ "${DOCKER_WATCHDOG_HEARTBEAT:-true}" == true ]]; then
|
||||
local hb_seconds=$(( ${DOCKER_WATCHDOG_HEARTBEAT_HOURS:-1} * 3600 ))
|
||||
local uptime_seconds=$(( CYCLE * DOCKER_WATCHDOG_INTERVAL ))
|
||||
if (( uptime_seconds % hb_seconds < DOCKER_WATCHDOG_INTERVAL )); then
|
||||
local uptime_hr=$(( uptime_seconds / 3600 ))
|
||||
info "♥ docker_watchdog alive — ~${uptime_hr}hr uptime ($(date '+%H:%M:%S'))"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Sleep until next cycle — interruptible by SIGTERM
|
||||
sleep "$SYSTEM_WATCHDOG_INTERVAL" &
|
||||
sleep "$DOCKER_WATCHDOG_INTERVAL" &
|
||||
wait $!
|
||||
|
||||
done
|
||||
Reference in New Issue
Block a user