446 lines
18 KiB
Bash
446 lines
18 KiB
Bash
#!/bin/bash
|
|
# -----------------------------------------------------------------------------------------------
|
|
# --------------------------------- Docker Watchdog --------------------------------------------
|
|
# -----------------------------------------------------------------------------------------------
|
|
# First line of defense — monitors Docker containers for memory, CPU, HTTP responsiveness,
|
|
# and unexpected stops. Restarts containers that exceed thresholds or go offline.
|
|
#
|
|
# Works alongside system_watchdog.sh:
|
|
# docker_watchdog.sh — container level, minimal disruption, tries to self-heal
|
|
# system_watchdog.sh — system level, last resort, reboots when healing fails
|
|
#
|
|
# Behaviour:
|
|
# Memory — immediate restart if hard limit exceeded
|
|
# CPU — strike system, restarts after CPU_FAIL_LIMIT consecutive hits
|
|
# HTTP — strike system, restarts after RESP_FAIL_LIMIT consecutive failures
|
|
# Required — strike system, restarts stopped containers, persistent skip list
|
|
# prevents reboot loops, auto-clears when container recovers
|
|
# Daemon — immediate notify if Docker daemon is unresponsive
|
|
#
|
|
# Strike cadence depends on cron schedule:
|
|
# Every 15min + 2 strikes = 30min sustained before restart
|
|
# Every 10min + 2 strikes = 20min sustained before restart
|
|
# Every 5min + 2 strikes = 10min sustained before restart
|
|
#
|
|
# All configuration in Master.conf under Docker Watchdog section.
|
|
# Supports --dry-run to show what would happen without acting.
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../Master.conf"
|
|
source "$SCRIPT_DIR/../common.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
TOTAL_CORES=$(nproc)
|
|
|
|
# Persistent skip list — shared with system_watchdog.sh
|
|
# Containers in this list are skipped until they recover
|
|
SKIP_LIST_FILE="$SYS_WATCHDOG_FAILED_FILE"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_GEAR Setup ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
success "Running as root"
|
|
info "$ICON_WATCHDOG Watchdog initialising — $TOTAL_CORES cores detected"
|
|
|
|
touch "$WATCHDOG_STATE_FILE" 2>/dev/null || {
|
|
error "Cannot create state file: $WATCHDOG_STATE_FILE"
|
|
exit 1
|
|
}
|
|
|
|
touch "$SKIP_LIST_FILE" 2>/dev/null || {
|
|
error "Cannot create skip list: $SKIP_LIST_FILE"
|
|
exit 1
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SUMMARY Status ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_WATCHDOG Monitored: ${!WATCHDOG_CONTAINERS[*]}"
|
|
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}"
|
|
echo "$ICON_MEM Soft mem: ${SOFT_MEM_THRESHOLD}% of limit"
|
|
echo "$ICON_ZFS CPU soft: ${SOFT_CPU_THRESHOLD}%"
|
|
echo "$ICON_ZFS CPU hard: ${HARD_CPU_THRESHOLD}%"
|
|
echo "$ICON_RETRY CPU strikes: ${CPU_FAIL_LIMIT}"
|
|
echo "$ICON_RETRY Resp strikes: ${RESP_FAIL_LIMIT}"
|
|
echo "$ICON_TIME Curl timeout: ${CURL_TIMEOUT}s"
|
|
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# STATE HELPERS
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
get_strikes() {
|
|
local container="$1" metric="$2"
|
|
grep -E "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f3
|
|
}
|
|
|
|
set_strikes() {
|
|
local container="$1" metric="$2" count="$3"
|
|
grep -vE "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null > "${WATCHDOG_STATE_FILE}.tmp"
|
|
echo "${container}:${metric}:${count}" >> "${WATCHDOG_STATE_FILE}.tmp"
|
|
mv "${WATCHDOG_STATE_FILE}.tmp" "$WATCHDOG_STATE_FILE"
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# SKIP LIST HELPERS
|
|
# Container skip list — persistent across reboots via /boot/
|
|
# Auto-clears entries when container is found running again.
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
is_in_skip_list() {
|
|
local container="$1"
|
|
grep -qE "^${container}$" "$SKIP_LIST_FILE" 2>/dev/null
|
|
}
|
|
|
|
add_to_skip_list() {
|
|
local container="$1"
|
|
if ! is_in_skip_list "$container"; then
|
|
echo "$container" >> "$SKIP_LIST_FILE"
|
|
warn "$ICON_WATCHDOG $container added to persistent skip list"
|
|
notify "$container added to watchdog skip list on $(hostname) — manual check recommended" "Docker Watchdog" "warning"
|
|
fi
|
|
}
|
|
|
|
remove_from_skip_list() {
|
|
local container="$1"
|
|
grep -vE "^${container}$" "$SKIP_LIST_FILE" 2>/dev/null > "${SKIP_LIST_FILE}.tmp"
|
|
mv "${SKIP_LIST_FILE}.tmp" "$SKIP_LIST_FILE"
|
|
success "$ICON_WATCHDOG $container recovered — removed from skip list"
|
|
notify "$container recovered and removed from watchdog skip list on $(hostname)" "Docker Watchdog" "normal"
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# SKIP LIST AUTO-HEAL CHECK
|
|
# On every run check if any skipped containers are now running.
|
|
# If running remove from skip list — could have recovered after reboot or manual fix.
|
|
# -----------------------------------------------------------------------------------------------
|
|
check_skip_list_recovery() {
|
|
[[ ! -s "$SKIP_LIST_FILE" ]] && return
|
|
|
|
info "$ICON_WATCHDOG Checking skip list for recovered containers..."
|
|
|
|
while IFS= read -r container; do
|
|
[[ -z "$container" ]] && continue
|
|
|
|
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
|
|
|
if [[ "$STATUS" == "true" ]]; then
|
|
remove_from_skip_list "$container"
|
|
else
|
|
log "$container still not running — remains on skip list"
|
|
fi
|
|
done < "$SKIP_LIST_FILE"
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# DOCKER DAEMON HEALTH CHECK
|
|
# Verifies Docker daemon is responding before attempting any container operations.
|
|
# A hung daemon means all checks will fail — notify immediately and exit.
|
|
# -----------------------------------------------------------------------------------------------
|
|
check_docker_daemon() {
|
|
info "$ICON_CONTAINERS Checking Docker daemon..."
|
|
|
|
if ! timeout 10 docker ps >/dev/null 2>&1; then
|
|
error "Docker daemon is not responding"
|
|
notify "Docker daemon unresponsive on $(hostname) — immediate attention required" "Docker Watchdog" "warning"
|
|
exit 1
|
|
fi
|
|
|
|
success "Docker daemon is healthy"
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# HELPERS
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
parse_stats() {
|
|
local container="$1"
|
|
docker stats --no-stream --format "{{.MemUsage}}|{{.CPUPerc}}" "$container"
|
|
}
|
|
|
|
convert_to_mb() {
|
|
local value="$1" unit="$2"
|
|
case "$unit" in
|
|
KiB) awk "BEGIN {print $value / 1024}" ;;
|
|
MiB) echo "$value" ;;
|
|
GiB) awk "BEGIN {print $value * 1024}" ;;
|
|
*) echo "UNKNOWN" ;;
|
|
esac
|
|
}
|
|
|
|
# Restarts a container and sends notification.
|
|
# Failed restarts also notify — system_watchdog.sh is the next line of defense.
|
|
restart_container() {
|
|
local container="$1" reason="$2"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would restart $container ($reason)"
|
|
return 0
|
|
fi
|
|
|
|
info "Restarting $container ($reason)..."
|
|
if docker restart "$container" >/dev/null 2>&1; then
|
|
echo "$ICON_STARTED $container restarted"
|
|
notify "$container restarted on $(hostname) — $reason" "Docker Watchdog" "warning"
|
|
return 0
|
|
else
|
|
error "Failed to restart $container"
|
|
notify "Failed to restart $container on $(hostname) — $reason" "Docker Watchdog" "warning"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# MEMORY CHECK
|
|
# Restarts immediately if container exceeds hard memory limit.
|
|
# Warns if approaching soft threshold.
|
|
# Usage: check_memory "Emby" 16384
|
|
# -----------------------------------------------------------------------------------------------
|
|
check_memory() {
|
|
local container="$1" limit_mb="$2"
|
|
local stats mem_raw mem_val mem_unit mem_mb usage_pct
|
|
|
|
stats=$(parse_stats "$container")
|
|
mem_raw=$(echo "$stats" | awk -F'|' '{print $1}' | awk '{print $1}')
|
|
mem_val=$(echo "$mem_raw" | sed -E 's/([0-9.]+).*/\1/')
|
|
mem_unit=$(echo "$mem_raw" | sed -E 's/[0-9.]+([a-zA-Z]+).*/\1/')
|
|
mem_mb=$(convert_to_mb "$mem_val" "$mem_unit")
|
|
local mem_int
|
|
mem_int=$(printf "%.0f" "$mem_mb")
|
|
usage_pct=$(( (mem_int * 100) / limit_mb ))
|
|
|
|
if (( usage_pct >= 100 )); then
|
|
error "$ICON_MEM $container memory ${mem_int}MB (${usage_pct}%) — exceeded ${limit_mb}MB hard limit"
|
|
restart_container "$container" "memory hard limit"
|
|
set_strikes "$container" "CPU" 0
|
|
set_strikes "$container" "RESP" 0
|
|
elif (( usage_pct >= SOFT_MEM_THRESHOLD )); then
|
|
warn "$ICON_MEM $container memory ${mem_int}MB (${usage_pct}% of ${limit_mb}MB limit)"
|
|
else
|
|
success "$ICON_MEM $container memory ${mem_int}MB (${usage_pct}% of ${limit_mb}MB limit)"
|
|
fi
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# CPU CHECK
|
|
# Strike system — restarts after CPU_FAIL_LIMIT consecutive over-threshold checks.
|
|
# Resets strikes on recovery or restart.
|
|
# Usage: check_cpu "Emby"
|
|
# -----------------------------------------------------------------------------------------------
|
|
check_cpu() {
|
|
local container="$1"
|
|
local stats cpu_raw cpu_norm cpu_int violations
|
|
|
|
stats=$(parse_stats "$container")
|
|
cpu_raw=$(echo "$stats" | awk -F'|' '{print $2}' | tr -d '%')
|
|
cpu_norm=$(awk "BEGIN {print $cpu_raw / $TOTAL_CORES}")
|
|
cpu_int=$(printf "%.0f" "$cpu_norm")
|
|
|
|
violations=$(get_strikes "$container" "CPU")
|
|
[[ -z "$violations" ]] && violations=0
|
|
|
|
if (( cpu_int >= HARD_CPU_THRESHOLD )); then
|
|
((violations++))
|
|
error "$ICON_ZFS $container CPU ${cpu_int}% — hard threshold ($violations/$CPU_FAIL_LIMIT strikes)"
|
|
set_strikes "$container" "CPU" "$violations"
|
|
elif (( cpu_int >= SOFT_CPU_THRESHOLD )); then
|
|
((violations++))
|
|
warn "$ICON_ZFS $container CPU ${cpu_int}% — soft threshold ($violations/$CPU_FAIL_LIMIT strikes)"
|
|
set_strikes "$container" "CPU" "$violations"
|
|
else
|
|
[[ $violations -gt 0 ]] && info "$ICON_ZFS $container CPU ${cpu_int}% — recovered, resetting strikes"
|
|
[[ $violations -eq 0 ]] && success "$ICON_ZFS $container CPU ${cpu_int}%"
|
|
set_strikes "$container" "CPU" 0
|
|
violations=0
|
|
fi
|
|
|
|
if (( violations >= CPU_FAIL_LIMIT )); then
|
|
error "$ICON_ZFS $container CPU limit hit for $CPU_FAIL_LIMIT consecutive checks"
|
|
restart_container "$container" "sustained CPU abuse"
|
|
set_strikes "$container" "CPU" 0
|
|
fi
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# RESPONSIVENESS CHECK
|
|
# Strike system — restarts after RESP_FAIL_LIMIT consecutive failed HTTP checks.
|
|
# Skips containers with no URL defined in WATCHDOG_CONTAINER_URLS.
|
|
# Usage: check_responsiveness "Emby"
|
|
# -----------------------------------------------------------------------------------------------
|
|
check_responsiveness() {
|
|
local container="$1"
|
|
local url="${WATCHDOG_CONTAINER_URLS[$container]:-}"
|
|
|
|
[[ -z "$url" ]] && return
|
|
|
|
local fails
|
|
fails=$(get_strikes "$container" "RESP")
|
|
[[ -z "$fails" ]] && fails=0
|
|
|
|
if ! curl -s --max-time "$CURL_TIMEOUT" "$url" >/dev/null 2>&1; then
|
|
((fails++))
|
|
warn "$ICON_PING $container unresponsive at $url ($fails/$RESP_FAIL_LIMIT strikes)"
|
|
set_strikes "$container" "RESP" "$fails"
|
|
else
|
|
[[ $fails -gt 0 ]] && info "$ICON_PING $container responsive again — resetting strikes"
|
|
[[ $fails -eq 0 ]] && success "$ICON_PING $container responsive at $url"
|
|
set_strikes "$container" "RESP" 0
|
|
fails=0
|
|
fi
|
|
|
|
if (( fails >= RESP_FAIL_LIMIT )); then
|
|
error "$ICON_PING $container unresponsive for $RESP_FAIL_LIMIT consecutive checks"
|
|
restart_container "$container" "HTTP unresponsive"
|
|
set_strikes "$container" "RESP" 0
|
|
fi
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# REQUIRED CONTAINER CHECK
|
|
# Monitors WATCHDOG_REQUIRED_CONTAINERS for unexpected stops.
|
|
# Strike system — attempts restart on each strike.
|
|
# After strike limit hit — adds to persistent skip list and notifies system_watchdog handoff.
|
|
# Skip list auto-clears at start of each run if container has recovered.
|
|
# Usage: check_required_containers
|
|
# -----------------------------------------------------------------------------------------------
|
|
check_required_containers() {
|
|
[[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -eq 0 ]] && return
|
|
|
|
info "$ICON_CONTAINERS Checking required containers..."
|
|
|
|
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
|
|
[[ -z "$container" ]] && continue
|
|
|
|
# Skip if on persistent skip list
|
|
if is_in_skip_list "$container"; then
|
|
warn "$ICON_NOT_RUNNING $container is on skip list — skipping until recovered"
|
|
continue
|
|
fi
|
|
|
|
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
|
|
|
if [[ "$STATUS" == "true" ]]; then
|
|
success "$ICON_RUNNING $container is running"
|
|
set_strikes "$container" "STOP" 0
|
|
continue
|
|
fi
|
|
|
|
if [[ "$STATUS" == "unknown" ]]; then
|
|
warn "$container not found on this host — skipping"
|
|
continue
|
|
fi
|
|
|
|
# Container is stopped — apply strike
|
|
local strikes
|
|
strikes=$(get_strikes "$container" "STOP")
|
|
[[ -z "$strikes" ]] && strikes=0
|
|
((strikes++))
|
|
|
|
warn "$ICON_NOT_RUNNING $container is stopped ($strikes/$SYS_WATCHDOG_STRIKE_LIMIT strikes)"
|
|
set_strikes "$container" "STOP" "$strikes"
|
|
|
|
# Attempt restart on each strike
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would attempt restart of $container"
|
|
else
|
|
if restart_container "$container" "unexpected stop"; then
|
|
set_strikes "$container" "STOP" 0
|
|
else
|
|
# Restart failed
|
|
if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then
|
|
error "$container failed to restart after $SYS_WATCHDOG_STRIKE_LIMIT attempts"
|
|
add_to_skip_list "$container"
|
|
set_strikes "$container" "STOP" 0
|
|
notify "$container handed off to system_watchdog on $(hostname) — added to skip list" "Docker Watchdog" "warning"
|
|
fi
|
|
fi
|
|
fi
|
|
done
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_WATCHDOG Watchdog Run ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_WATCHDOG Watchdog Run — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
SKIPPED=()
|
|
|
|
# Daemon check first — if daemon is down nothing else works
|
|
check_docker_daemon
|
|
|
|
# Auto-heal skip list before processing
|
|
check_skip_list_recovery
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# Resource monitoring — WATCHDOG_CONTAINERS
|
|
# -----------------------------------------------------------------------------------------------
|
|
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_MEM Resource Monitoring ━━━"
|
|
|
|
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
|
|
echo ""
|
|
info "$ICON_CONTAINERS $container"
|
|
|
|
if ! docker inspect "$container" &>/dev/null; then
|
|
warn "$container not found — skipping"
|
|
SKIPPED+=("$container")
|
|
continue
|
|
fi
|
|
|
|
if ! docker ps --filter "name=^/${container}$" --format "{{.Names}}" | grep -qw "$container"; then
|
|
warn "$ICON_NOT_RUNNING $container is not running — skipping resource checks"
|
|
SKIPPED+=("$container")
|
|
continue
|
|
fi
|
|
|
|
check_memory "$container" "${WATCHDOG_CONTAINERS[$container]}"
|
|
check_cpu "$container"
|
|
check_responsiveness "$container"
|
|
done
|
|
fi
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# Required container monitoring — WATCHDOG_REQUIRED_CONTAINERS
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_CONTAINERS Required Container Check ━━━"
|
|
check_required_containers
|
|
|
|
END=$(date +%s)
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SUMMARY Summary ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY WATCHDOG SUMMARY ━━━━━"
|
|
echo "$ICON_TIME $(date '+%Y-%m-%d %H:%M:%S')"
|
|
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
|
echo "$ICON_WATCHDOG Monitored: ${#WATCHDOG_CONTAINERS[@]} containers"
|
|
echo "$ICON_CONTAINERS Required: ${#WATCHDOG_REQUIRED_CONTAINERS[@]} containers"
|
|
[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]}"
|
|
[[ "$DRY_RUN" == true ]] && echo "$ICON_WARN Dry Run: no actions taken"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |