diff --git a/Docker_Essentials/docker_watchdog.sh b/Docker_Essentials/docker_watchdog.sh index fba76cc..6e30075 100644 --- a/Docker_Essentials/docker_watchdog.sh +++ b/Docker_Essentials/docker_watchdog.sh @@ -2,28 +2,35 @@ # ----------------------------------------------------------------------------------------------- # --------------------------------- Docker Watchdog -------------------------------------------- # ----------------------------------------------------------------------------------------------- -# First line of defense — monitors Docker containers for memory, CPU, HTTP responsiveness, -# and unexpected stops. Restarts containers that exceed thresholds or go offline. +# Two-tier self-healing container monitoring system. # -# Works alongside system_watchdog.sh: -# docker_watchdog.sh — container level, minimal disruption, tries to self-heal -# system_watchdog.sh — system level, last resort, reboots when healing fails +# Tier 1 — Strict monitoring (configured containers only) +# Memory hard limits — immediate restart if exceeded +# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT strikes +# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT strikes +# Required containers — must always be running, strike system with skip list # -# Behaviour: -# Memory — immediate restart if hard limit exceeded -# CPU — strike system, restarts after CPU_FAIL_LIMIT consecutive hits -# HTTP — strike system, restarts after RESP_FAIL_LIMIT consecutive failures -# Required — strike system, restarts stopped containers, persistent skip list -# prevents reboot loops, auto-clears when container recovers -# Daemon — immediate notify if Docker daemon is unresponsive +# Tier 2 — Global health scan (all running containers) +# Unhealthy status — Docker HEALTHCHECK unhealthy → restart +# OOM killed — kernel killed container → restart + notify +# Crash loop detection — RestartCount climbing → notify, critical above limit +# Dead containers — remove and restart +# Unexpected exits — non-zero exit code → restart # -# Strike cadence depends on cron schedule: -# Every 15min + 2 strikes = 30min sustained before restart -# Every 10min + 2 strikes = 20min sustained before restart -# Every 5min + 2 strikes = 10min sustained before restart +# Cross-cutting intelligence (applies to both tiers): +# Startup grace period — skip restarts while system is still booting +# Dependency ordering — restart database before app, not the other way around +# Restart loop protect — stop restarting after X restarts in X hours → skip list +# Skip list auto-clear — clears when container recovers, fresh start +# Notification batching — one clean summary per run, not one ping per event +# +# State files: +# WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot) +# SYS_WATCHDOG_FAILED_FILE — persistent skip list (/boot — survives reboots) +# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (/boot) # # All configuration in Master.conf under Docker Watchdog section. -# Supports --dry-run to show what would happen without acting. +# Supports --dry-run and --status. # ----------------------------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -33,12 +40,6 @@ source "$SCRIPT_DIR/../common.sh" parse_args "$@" -TOTAL_CORES=$(nproc) - -# Persistent skip list — shared with system_watchdog.sh -# Containers in this list are skipped until they recover -SKIP_LIST_FILE="$SYS_WATCHDOG_FAILED_FILE" - # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Setup ━━━ # ----------------------------------------------------------------------------------------------- @@ -51,17 +52,27 @@ if [[ "$EUID" -ne 0 ]]; then fi success "Running as root" -info "$ICON_WATCHDOG Watchdog initialising — $TOTAL_CORES cores detected" -touch "$WATCHDOG_STATE_FILE" 2>/dev/null || { - error "Cannot create state file: $WATCHDOG_STATE_FILE" +if ! command -v docker >/dev/null 2>&1; then + error "Docker not found" exit 1 -} +fi -touch "$SKIP_LIST_FILE" 2>/dev/null || { - error "Cannot create skip list: $SKIP_LIST_FILE" - exit 1 -} +success "Docker found" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" + +# State files +touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \ + "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null + +# Build ignore map for Tier 2 +declare -A IGNORE_MAP +for c in "${WATCHDOG_SCAN_IGNORE[@]}"; do + [[ -n "$c" ]] && IGNORE_MAP["$c"]=1 +done + +# Batch notification collector +NOTIFY_EVENTS=() # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Status ━━━ @@ -69,378 +80,577 @@ touch "$SKIP_LIST_FILE" 2>/dev/null || { if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_WATCHDOG Monitored: ${!WATCHDOG_CONTAINERS[*]}" - echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}" - echo "$ICON_MEM Soft mem: ${SOFT_MEM_THRESHOLD}% of limit" - echo "$ICON_ZFS CPU soft: ${SOFT_CPU_THRESHOLD}%" - echo "$ICON_ZFS CPU hard: ${HARD_CPU_THRESHOLD}%" - echo "$ICON_RETRY CPU strikes: ${CPU_FAIL_LIMIT}" - echo "$ICON_RETRY Resp strikes: ${RESP_FAIL_LIMIT}" - echo "$ICON_TIME Curl timeout: ${CURL_TIMEOUT}s" - echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_CONTAINERS Watched: ${!WATCHDOG_CONTAINERS[@]}" + echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}" + echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL" + echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s" + echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" + echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY" + echo "$ICON_WATCHDOG Ignore list: ${WATCHDOG_SCAN_IGNORE[*]:-none}" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + UPTIME_S=$(awk '{print int($1)}' /proc/uptime) + echo "$ICON_TIME System uptime: $(format_duration $UPTIME_S)" + [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]] && \ + warn "Within startup grace period — restarts suppressed" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" - -# ----------------------------------------------------------------------------------------------- -# STATE HELPERS -# ----------------------------------------------------------------------------------------------- - -get_strikes() { - local container="$1" metric="$2" - grep -E "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f3 -} - -set_strikes() { - local container="$1" metric="$2" count="$3" - grep -vE "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null > "${WATCHDOG_STATE_FILE}.tmp" - echo "${container}:${metric}:${count}" >> "${WATCHDOG_STATE_FILE}.tmp" - mv "${WATCHDOG_STATE_FILE}.tmp" "$WATCHDOG_STATE_FILE" -} - -# ----------------------------------------------------------------------------------------------- -# SKIP LIST HELPERS -# Container skip list — persistent across reboots via /boot/ -# Auto-clears entries when container is found running again. -# ----------------------------------------------------------------------------------------------- - -is_in_skip_list() { - local container="$1" - grep -qE "^${container}$" "$SKIP_LIST_FILE" 2>/dev/null -} - -add_to_skip_list() { - local container="$1" - if ! is_in_skip_list "$container"; then - echo "$container" >> "$SKIP_LIST_FILE" - warn "$ICON_WATCHDOG $container added to persistent skip list" - notify "$container added to watchdog skip list on $(hostname) — manual check recommended" "Docker Watchdog" "warning" - fi -} - -remove_from_skip_list() { - local container="$1" - grep -vE "^${container}$" "$SKIP_LIST_FILE" 2>/dev/null > "${SKIP_LIST_FILE}.tmp" - mv "${SKIP_LIST_FILE}.tmp" "$SKIP_LIST_FILE" - success "$ICON_WATCHDOG $container recovered — removed from skip list" - notify "$container recovered and removed from watchdog skip list on $(hostname)" "Docker Watchdog" "normal" -} - -# ----------------------------------------------------------------------------------------------- -# SKIP LIST AUTO-HEAL CHECK -# On every run check if any skipped containers are now running. -# If running remove from skip list — could have recovered after reboot or manual fix. -# ----------------------------------------------------------------------------------------------- -check_skip_list_recovery() { - [[ ! -s "$SKIP_LIST_FILE" ]] && return - - info "$ICON_WATCHDOG Checking skip list for recovered containers..." - - while IFS= read -r container; do - [[ -z "$container" ]] && continue - - STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") - - if [[ "$STATUS" == "true" ]]; then - remove_from_skip_list "$container" - else - log "$container still not running — remains on skip list" - fi - done < "$SKIP_LIST_FILE" -} - -# ----------------------------------------------------------------------------------------------- -# DOCKER DAEMON HEALTH CHECK -# Verifies Docker daemon is responding before attempting any container operations. -# A hung daemon means all checks will fail — notify immediately and exit. -# ----------------------------------------------------------------------------------------------- -check_docker_daemon() { - info "$ICON_CONTAINERS Checking Docker daemon..." - - if ! timeout 10 docker ps >/dev/null 2>&1; then - error "Docker daemon is not responding" - notify "Docker daemon unresponsive on $(hostname) — immediate attention required" "Docker Watchdog" "warning" - exit 1 - fi - - success "Docker daemon is healthy" -} - # ----------------------------------------------------------------------------------------------- # HELPERS # ----------------------------------------------------------------------------------------------- -parse_stats() { - local container="$1" - docker stats --no-stream --format "{{.MemUsage}}|{{.CPUPerc}}" "$container" +# Strike count management +get_strikes() { + grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0" } -convert_to_mb() { - local value="$1" unit="$2" - case "$unit" in - KiB) awk "BEGIN {print $value / 1024}" ;; - MiB) echo "$value" ;; - GiB) awk "BEGIN {print $value * 1024}" ;; - *) echo "UNKNOWN" ;; - esac +set_strikes() { + local container="$1" count="$2" file="$3" + if grep -q "^${container}:" "$file" 2>/dev/null; then + sed -i "s/^${container}:.*/${container}:${count}/" "$file" + else + echo "${container}:${count}" >> "$file" + fi } -# Restarts a container and sends notification. -# Failed restarts also notify — system_watchdog.sh is the next line of defense. -restart_container() { +# Persistent skip list management +is_skipped() { + grep -q "^${1}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null +} + +add_to_skip_list() { local container="$1" reason="$2" + if ! is_skipped "$container"; then + echo "$container" >> "$SYS_WATCHDOG_FAILED_FILE" + error "$container added to skip list — $reason" + queue_notify "$container added to skip list on $(hostname) — $reason — manual intervention needed" "critical" + fi +} + +remove_from_skip_list() { + sed -i "/^${1}$/d" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null + success "$1 removed from skip list — recovered" +} + +# Restart loop tracking — /boot/ bounded file, auto-purges old entries +log_restart() { + local container="$1" + local now + now=$(date '+%Y-%m-%d %H:%M:%S') + local cutoff + cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S') + + # Append this restart + echo "${container}|${now}" >> "$WATCHDOG_CONTAINER_RESTART_LOG" + + # Purge entries older than window — keeps file bounded + local tmp="${WATCHDOG_CONTAINER_RESTART_LOG}.tmp" + awk -F'|' -v cutoff="$cutoff" '$2 >= cutoff' \ + "$WATCHDOG_CONTAINER_RESTART_LOG" > "$tmp" && \ + mv "$tmp" "$WATCHDOG_CONTAINER_RESTART_LOG" +} + +# Count restarts for a container within the current window +get_restart_count() { + local container="$1" + local cutoff + cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S') + awk -F'|' -v c="$container" -v cutoff="$cutoff" \ + '$1==c && $2>=cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l +} + +# Dependency check — returns 0 if all dependencies are running +dependencies_satisfied() { + local container="$1" + local deps="${WATCHDOG_DEPENDENCIES[$container]:-}" + [[ -z "$deps" ]] && return 0 + + for dep in $deps; do + local status + status=$(docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null) + if [[ "$status" != "true" ]]; then + warn "$container — dependency $dep is not running — skipping restart this cycle" + return 1 + fi + done + return 0 +} + +# Restart a container with restart loop protection +# Returns 0 on success, 1 on failure, 2 if loop limit hit +safe_restart() { + local container="$1" reason="$2" + + # Check restart loop + local restart_count + restart_count=$(get_restart_count "$container") + + if [[ "$restart_count" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then + add_to_skip_list "$container" \ + "restarted $restart_count times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" + return 2 + fi + + # Check dependencies + dependencies_satisfied "$container" || return 1 + + # Check startup grace + local uptime_s + uptime_s=$(awk '{print int($1)}' /proc/uptime) + if [[ "$uptime_s" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then + warn "$container — within startup grace period (${uptime_s}s / ${WATCHDOG_STARTUP_GRACE}s) — skipping restart" + return 1 + fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would restart $container ($reason)" return 0 fi - info "Restarting $container ($reason)..." + info "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]..." if docker restart "$container" >/dev/null 2>&1; then - echo "$ICON_STARTED $container restarted" - notify "$container restarted on $(hostname) — $reason" "Docker Watchdog" "warning" + success "$ICON_STARTED $container restarted" + log_restart "$container" return 0 else error "Failed to restart $container" - notify "Failed to restart $container on $(hostname) — $reason" "Docker Watchdog" "warning" return 1 fi } -# ----------------------------------------------------------------------------------------------- -# MEMORY CHECK -# Restarts immediately if container exceeds hard memory limit. -# Warns if approaching soft threshold. -# Usage: check_memory "Emby" 16384 -# ----------------------------------------------------------------------------------------------- -check_memory() { - local container="$1" limit_mb="$2" - local stats mem_raw mem_val mem_unit mem_mb usage_pct +# Queue a notification event for batching +queue_notify() { + local message="$1" severity="${2:-warning}" + NOTIFY_EVENTS+=("${severity}|${message}") + log "Queued: $message" +} - stats=$(parse_stats "$container") - mem_raw=$(echo "$stats" | awk -F'|' '{print $1}' | awk '{print $1}') - mem_val=$(echo "$mem_raw" | sed -E 's/([0-9.]+).*/\1/') - mem_unit=$(echo "$mem_raw" | sed -E 's/[0-9.]+([a-zA-Z]+).*/\1/') - mem_mb=$(convert_to_mb "$mem_val" "$mem_unit") - local mem_int - mem_int=$(printf "%.0f" "$mem_mb") - usage_pct=$(( (mem_int * 100) / limit_mb )) +# Send all queued notifications +flush_notify() { + [[ ${#NOTIFY_EVENTS[@]} -eq 0 ]] && return - if (( usage_pct >= 100 )); then - error "$ICON_MEM $container memory ${mem_int}MB (${usage_pct}%) — exceeded ${limit_mb}MB hard limit" - restart_container "$container" "memory hard limit" - set_strikes "$container" "CPU" 0 - set_strikes "$container" "RESP" 0 - elif (( usage_pct >= SOFT_MEM_THRESHOLD )); then - warn "$ICON_MEM $container memory ${mem_int}MB (${usage_pct}% of ${limit_mb}MB limit)" + if [[ "$WATCHDOG_BATCH_NOTIFY" == "true" ]]; then + # Build single batched message + local critical_count=0 warning_count=0 + local messages=() + local highest_severity="normal" + + for event in "${NOTIFY_EVENTS[@]}"; do + local sev="${event%%|*}" + local msg="${event#*|}" + messages+=("$msg") + [[ "$sev" == "critical" ]] && ((critical_count++)) && highest_severity="warning" + [[ "$sev" == "warning" ]] && ((warning_count++)) && \ + [[ "$highest_severity" == "normal" ]] && highest_severity="warning" + done + + local summary + summary=$(printf '%s. ' "${messages[@]}") + notify "Docker Watchdog on $(hostname) — ${#NOTIFY_EVENTS[@]} event(s): $summary" \ + "Docker Watchdog" "$highest_severity" else - success "$ICON_MEM $container memory ${mem_int}MB (${usage_pct}% of ${limit_mb}MB limit)" + # Send individual notifications + for event in "${NOTIFY_EVENTS[@]}"; do + local sev="${event%%|*}" + local msg="${event#*|}" + [[ "$sev" == "critical" ]] && sev="warning" + notify "$msg" "Docker Watchdog" "$sev" + done fi + + NOTIFY_EVENTS=() } # ----------------------------------------------------------------------------------------------- -# CPU CHECK -# Strike system — restarts after CPU_FAIL_LIMIT consecutive over-threshold checks. -# Resets strikes on recovery or restart. -# Usage: check_cpu "Emby" +# STARTUP GRACE CHECK # ----------------------------------------------------------------------------------------------- -check_cpu() { - local container="$1" - local stats cpu_raw cpu_norm cpu_int violations +UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime) +IN_GRACE_PERIOD=false - stats=$(parse_stats "$container") - cpu_raw=$(echo "$stats" | awk -F'|' '{print $2}' | tr -d '%') - cpu_norm=$(awk "BEGIN {print $cpu_raw / $TOTAL_CORES}") - cpu_int=$(printf "%.0f" "$cpu_norm") - - violations=$(get_strikes "$container" "CPU") - [[ -z "$violations" ]] && violations=0 - - if (( cpu_int >= HARD_CPU_THRESHOLD )); then - ((violations++)) - error "$ICON_ZFS $container CPU ${cpu_int}% — hard threshold ($violations/$CPU_FAIL_LIMIT strikes)" - set_strikes "$container" "CPU" "$violations" - elif (( cpu_int >= SOFT_CPU_THRESHOLD )); then - ((violations++)) - warn "$ICON_ZFS $container CPU ${cpu_int}% — soft threshold ($violations/$CPU_FAIL_LIMIT strikes)" - set_strikes "$container" "CPU" "$violations" - else - [[ $violations -gt 0 ]] && info "$ICON_ZFS $container CPU ${cpu_int}% — recovered, resetting strikes" - [[ $violations -eq 0 ]] && success "$ICON_ZFS $container CPU ${cpu_int}%" - set_strikes "$container" "CPU" 0 - violations=0 - fi - - if (( violations >= CPU_FAIL_LIMIT )); then - error "$ICON_ZFS $container CPU limit hit for $CPU_FAIL_LIMIT consecutive checks" - restart_container "$container" "sustained CPU abuse" - set_strikes "$container" "CPU" 0 - fi -} +if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then + GRACE_REMAINING=$(( WATCHDOG_STARTUP_GRACE - UPTIME_SECONDS )) + warn "System uptime $(format_duration $UPTIME_SECONDS) — startup grace period active (${GRACE_REMAINING}s remaining)" + warn "Checks will run but restarts are suppressed during grace period" + IN_GRACE_PERIOD=true +else + info "System uptime $(format_duration $UPTIME_SECONDS) — grace period passed" +fi # ----------------------------------------------------------------------------------------------- -# RESPONSIVENESS CHECK -# Strike system — restarts after RESP_FAIL_LIMIT consecutive failed HTTP checks. -# Skips containers with no URL defined in WATCHDOG_CONTAINER_URLS. -# Usage: check_responsiveness "Emby" +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# TIER 1 — Strict Monitoring +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # ----------------------------------------------------------------------------------------------- -check_responsiveness() { - local container="$1" - local url="${WATCHDOG_CONTAINER_URLS[$container]:-}" - [[ -z "$url" ]] && return +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " $ICON_WATCHDOG TIER 1 — Strict Monitoring" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - local fails - fails=$(get_strikes "$container" "RESP") - [[ -z "$fails" ]] && fails=0 +T1_RESTARTS=0 +T1_WARNINGS=0 - if ! curl -s --max-time "$CURL_TIMEOUT" "$url" >/dev/null 2>&1; then - ((fails++)) - warn "$ICON_PING $container unresponsive at $url ($fails/$RESP_FAIL_LIMIT strikes)" - set_strikes "$container" "RESP" "$fails" - else - [[ $fails -gt 0 ]] && info "$ICON_PING $container responsive again — resetting strikes" - [[ $fails -eq 0 ]] && success "$ICON_PING $container responsive at $url" - set_strikes "$container" "RESP" 0 - fails=0 - fi - - if (( fails >= RESP_FAIL_LIMIT )); then - error "$ICON_PING $container unresponsive for $RESP_FAIL_LIMIT consecutive checks" - restart_container "$container" "HTTP unresponsive" - set_strikes "$container" "RESP" 0 - fi -} - -# ----------------------------------------------------------------------------------------------- -# REQUIRED CONTAINER CHECK -# Monitors WATCHDOG_REQUIRED_CONTAINERS for unexpected stops. -# Strike system — attempts restart on each strike. -# After strike limit hit — adds to persistent skip list and notifies system_watchdog handoff. -# Skip list auto-clears at start of each run if container has recovered. -# Usage: check_required_containers -# ----------------------------------------------------------------------------------------------- -check_required_containers() { - [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -eq 0 ]] && return - - info "$ICON_CONTAINERS Checking required containers..." +# ── Required Containers ────────────────────────────────────────────────────────────────────── +if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_CONTAINERS Required Containers ━━━" for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do [[ -z "$container" ]] && continue - # Skip if on persistent skip list - if is_in_skip_list "$container"; then - warn "$ICON_NOT_RUNNING $container is on skip list — skipping until recovered" + STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + + # Check skip list + if is_skipped "$container"; then + if [[ "$STATUS" == "true" ]]; then + remove_from_skip_list "$container" + set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" + # Clear restart history on recovery + sed -i "/^${container}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null + queue_notify "$container recovered on $(hostname) — removed from skip list" "normal" + else + warn "$ICON_NOT_RUNNING $container — on skip list, manual intervention needed" + RESTART_COUNT=$(get_restart_count "$container") + warn " Restarted $RESTART_COUNT time(s) in last ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" + fi continue fi - STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") - if [[ "$STATUS" == "true" ]]; then - success "$ICON_RUNNING $container is running" - set_strikes "$container" "STOP" 0 - continue - fi - - if [[ "$STATUS" == "unknown" ]]; then - warn "$container not found on this host — skipping" - continue - fi - - # Container is stopped — apply strike - local strikes - strikes=$(get_strikes "$container" "STOP") - [[ -z "$strikes" ]] && strikes=0 - ((strikes++)) - - warn "$ICON_NOT_RUNNING $container is stopped ($strikes/$SYS_WATCHDOG_STRIKE_LIMIT strikes)" - set_strikes "$container" "STOP" "$strikes" - - # Attempt restart on each strike - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would attempt restart of $container" + success "$ICON_RUNNING $container — running" + set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" else - if restart_container "$container" "unexpected stop"; then - set_strikes "$container" "STOP" 0 - else - # Restart failed - if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then - error "$container failed to restart after $SYS_WATCHDOG_STRIKE_LIMIT attempts" - add_to_skip_list "$container" - set_strikes "$container" "STOP" 0 - notify "$container handed off to system_watchdog on $(hostname) — added to skip list" "Docker Watchdog" "warning" - fi + STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE") + STRIKES=$(( STRIKES + 1 )) + set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE" + warn "$ICON_NOT_RUNNING $container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)" + ((T1_WARNINGS++)) + + if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then + result=0 + safe_restart "$container" "required container down" || result=$? + case $result in + 0) + set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" + ((T1_RESTARTS++)) + queue_notify "$container was down and restarted on $(hostname)" "warning" + ;; + 2) + : # already added to skip list in safe_restart + ;; + *) + queue_notify "$container failed to restart on $(hostname) — check dependencies and logs" "warning" + ;; + esac fi fi done -} +fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_WATCHDOG Watchdog Run ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_WATCHDOG Watchdog Run — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "" - -START=$(date +%s) -SKIPPED=() - -# Daemon check first — if daemon is down nothing else works -check_docker_daemon - -# Auto-heal skip list before processing -check_skip_list_recovery - -# ----------------------------------------------------------------------------------------------- -# Resource monitoring — WATCHDOG_CONTAINERS -# ----------------------------------------------------------------------------------------------- +# ── Memory and CPU Monitoring ──────────────────────────────────────────────────────────────── if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then echo "" - echo "━━━ $ICON_MEM Resource Monitoring ━━━" + echo "━━━ $ICON_MEM Memory / CPU Monitoring ━━━" + + STATS=$(docker stats --no-stream \ + --format "{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}" 2>/dev/null) + TOTAL_CORES=$(nproc 2>/dev/null || echo 1) for container in "${!WATCHDOG_CONTAINERS[@]}"; do - echo "" - info "$ICON_CONTAINERS $container" + MEM_LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}" + CONTAINER_STATS=$(echo "$STATS" | grep "^${container}|" | head -1) + [[ -z "$CONTAINER_STATS" ]] && continue - if ! docker inspect "$container" &>/dev/null; then - warn "$container not found — skipping" - SKIPPED+=("$container") - continue + MEM_USAGE=$(echo "$CONTAINER_STATS" | cut -d'|' -f2 | awk '{print $1}') + MEM_UNIT=$(echo "$MEM_USAGE" | grep -oE '[A-Za-z]+') + MEM_VALUE=$(echo "$MEM_USAGE" | grep -oE '[0-9.]+') + + case "$MEM_UNIT" in + GiB|GB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE * 1024}") ;; + MiB|MB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE}") ;; + KiB|KB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE / 1024}") ;; + *) MEM_MB=0 ;; + esac + + MEM_PCT=$(awk "BEGIN {printf \"%.0f\", $MEM_MB * 100 / $MEM_LIMIT_MB}") + CPU_RAW=$(echo "$CONTAINER_STATS" | cut -d'|' -f3 | tr -d '%') + CPU_NORM=$(awk "BEGIN {printf \"%.1f\", $CPU_RAW / $TOTAL_CORES}") + CPU_INT=$(printf "%.0f" "$CPU_NORM") + + # Memory check + if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then + error "$container — memory ${MEM_VALUE}${MEM_UNIT} exceeded hard limit ${MEM_LIMIT_MB}MB" + safe_restart "$container" "memory hard limit exceeded" + ((T1_RESTARTS++)) + queue_notify "$container exceeded memory limit on $(hostname) — ${MEM_VALUE}${MEM_UNIT} / ${MEM_LIMIT_MB}MB — restarted" "warning" + elif [[ "$MEM_PCT" -ge "$SOFT_MEM_THRESHOLD" ]]; then + warn "$container — memory ${MEM_VALUE}${MEM_UNIT} (${MEM_PCT}% of ${MEM_LIMIT_MB}MB)" + ((T1_WARNINGS++)) + else + success "$container — memory ${MEM_VALUE}${MEM_UNIT} (${MEM_PCT}%) CPU ${CPU_NORM}%" fi - if ! docker ps --filter "name=^/${container}$" --format "{{.Names}}" | grep -qw "$container"; then - warn "$ICON_NOT_RUNNING $container is not running — skipping resource checks" - SKIPPED+=("$container") - continue + # CPU check + if [[ "$CPU_INT" -ge "$HARD_CPU_THRESHOLD" ]]; then + CPU_STRIKES=$(get_strikes "${container}_cpu" "$WATCHDOG_STATE_FILE") + CPU_STRIKES=$(( CPU_STRIKES + 1 )) + set_strikes "${container}_cpu" "$CPU_STRIKES" "$WATCHDOG_STATE_FILE" + warn "$container — CPU ${CPU_NORM}% (strike $CPU_STRIKES/$CPU_FAIL_LIMIT)" + if [[ "$CPU_STRIKES" -ge "$CPU_FAIL_LIMIT" ]]; then + safe_restart "$container" "CPU threshold exceeded" + set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE" + ((T1_RESTARTS++)) + queue_notify "$container CPU ${CPU_NORM}% on $(hostname) — restarted after $CPU_FAIL_LIMIT strikes" "warning" + fi + else + set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE" fi + done +fi - check_memory "$container" "${WATCHDOG_CONTAINERS[$container]}" - check_cpu "$container" - check_responsiveness "$container" +# ── HTTP Responsiveness ────────────────────────────────────────────────────────────────────── +if [[ ${#WATCHDOG_CONTAINER_URLS[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_NET HTTP Responsiveness ━━━" + + for container in "${!WATCHDOG_CONTAINER_URLS[@]}"; do + URL="${WATCHDOG_CONTAINER_URLS[$container]}" + if curl -sf --max-time "$CURL_TIMEOUT" "$URL" >/dev/null 2>&1; then + success "$container — responding at $URL" + set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE" + else + HTTP_STRIKES=$(get_strikes "${container}_http" "$WATCHDOG_STATE_FILE") + HTTP_STRIKES=$(( HTTP_STRIKES + 1 )) + set_strikes "${container}_http" "$HTTP_STRIKES" "$WATCHDOG_STATE_FILE" + warn "$container — not responding at $URL (strike $HTTP_STRIKES/$RESP_FAIL_LIMIT)" + ((T1_WARNINGS++)) + if [[ "$HTTP_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then + safe_restart "$container" "HTTP unresponsive" + set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE" + ((T1_RESTARTS++)) + queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning" + fi + fi done fi # ----------------------------------------------------------------------------------------------- -# Required container monitoring — WATCHDOG_REQUIRED_CONTAINERS +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# TIER 2 — Global Health Scan +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_CONTAINERS Required Container Check ━━━" -check_required_containers -END=$(date +%s) +T2_RESTARTS=0 +T2_WARNINGS=0 + +if [[ "$WATCHDOG_SCAN_ALL" != "true" ]]; then + info "Global scan disabled (WATCHDOG_SCAN_ALL=false) — skipping Tier 2" +else + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " $ICON_WATCHDOG TIER 2 — Global Health Scan" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + ALL_CONTAINERS=$(docker ps --format "{{.Names}}" 2>/dev/null) + + # ── Unhealthy containers ───────────────────────────────────────────────────────────────── + if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then + echo "" + echo "━━━ $ICON_HEALTH Unhealthy Status ━━━" + UNHEALTHY=$(docker ps --filter health=unhealthy --format "{{.Names}}" 2>/dev/null) + + if [[ -z "$UNHEALTHY" ]]; then + success "No unhealthy containers" + else + while IFS= read -r container; do + [[ -z "$container" ]] && continue + [[ -n "${IGNORE_MAP[$container]:-}" ]] && \ + { info "Skipping $container (ignore list)"; continue; } + is_skipped "$container" && \ + { warn "$container — unhealthy but on skip list, manual intervention needed"; continue; } + + error "$container — Docker health status: unhealthy" + ((T2_WARNINGS++)) + result=0 + safe_restart "$container" "unhealthy health status" || result=$? + if [[ $result -eq 0 ]]; then + ((T2_RESTARTS++)) + queue_notify "$container unhealthy on $(hostname) — restarted" "warning" + elif [[ $result -eq 2 ]]; then + : # skip list message already queued + fi + done <<< "$UNHEALTHY" + fi + fi + + # ── OOM killed containers ──────────────────────────────────────────────────────────────── + if [[ "$WATCHDOG_NOTIFY_OOM" == "true" ]]; then + echo "" + echo "━━━ $ICON_MEM OOM Kill Check ━━━" + OOM_FOUND=false + + while IFS= read -r container; do + [[ -z "$container" ]] && continue + [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue + is_skipped "$container" && continue + + OOM=$(docker inspect -f '{{.State.OOMKilled}}' "$container" 2>/dev/null) + if [[ "$OOM" == "true" ]]; then + error "$container — OOM killed by kernel" + OOM_FOUND=true + ((T2_WARNINGS++)) + result=0 + safe_restart "$container" "OOM killed" || result=$? + if [[ $result -eq 0 ]]; then + ((T2_RESTARTS++)) + queue_notify "$container OOM killed on $(hostname) — system under memory pressure — restarted" "warning" + fi + fi + done <<< "$ALL_CONTAINERS" + + [[ "$OOM_FOUND" == false ]] && success "No OOM kills detected" + fi + + # ── Crash loop detection ───────────────────────────────────────────────────────────────── + if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then + echo "" + echo "━━━ $ICON_REBOOT_SMART Crash Loop Detection ━━━" + CRASH_FOUND=false + + while IFS= read -r container; do + [[ -z "$container" ]] && continue + [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue + is_skipped "$container" && continue + + RESTART_COUNT=$(docker inspect -f '{{.RestartCount}}' "$container" 2>/dev/null || echo 0) + PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d: -f2 || echo 0) + + # Update baseline + set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE" + + if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then + CRASH_FOUND=true + ((T2_WARNINGS++)) + if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then + error "$container — crash loop CRITICAL: $RESTART_COUNT total Docker restarts" + queue_notify "$container crash loop CRITICAL on $(hostname) — $RESTART_COUNT Docker restarts — manual intervention needed" "critical" + else + warn "$container — Docker restarted since last check (total: $RESTART_COUNT)" + queue_notify "$container restarted on $(hostname) — Docker restart count: $RESTART_COUNT" "warning" + fi + elif [[ "$RESTART_COUNT" -gt 0 ]]; then + log "$container — Docker restart count: $RESTART_COUNT (stable)" + fi + done <<< "$ALL_CONTAINERS" + + [[ "$CRASH_FOUND" == false ]] && success "No crash loops detected" + fi + + # ── Dead containers ────────────────────────────────────────────────────────────────────── + if [[ "$WATCHDOG_RESTART_DEAD" == "true" ]]; then + echo "" + echo "━━━ $ICON_STOPPED Dead Containers ━━━" + DEAD=$(docker ps -a --filter status=dead --format "{{.Names}}" 2>/dev/null) + + if [[ -z "$DEAD" ]]; then + success "No dead containers" + else + while IFS= read -r container; do + [[ -z "$container" ]] && continue + [[ -n "${IGNORE_MAP[$container]:-}" ]] && \ + { info "Skipping $container (ignore list)"; continue; } + is_skipped "$container" && \ + { warn "$container — dead but on skip list, manual intervention needed"; continue; } + + error "$container — status: dead" + ((T2_WARNINGS++)) + RESTART_COUNT=$(get_restart_count "$container") + if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then + add_to_skip_list "$container" "dead state — restarted $RESTART_COUNT times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" + elif [[ "$DRY_RUN" == false ]]; then + docker rm "$container" >/dev/null 2>&1 + if docker start "$container" >/dev/null 2>&1; then + success "$container removed from dead state and restarted" + log_restart "$container" + ((T2_RESTARTS++)) + queue_notify "$container was dead on $(hostname) — removed and restarted" "warning" + else + error "$container failed to restart after dead state" + queue_notify "$container dead and failed to restart on $(hostname) — manual intervention needed" "warning" + fi + else + warn "DRY RUN — would remove and restart $container" + fi + done <<< "$DEAD" + fi + fi + + # ── Unexpected exits ───────────────────────────────────────────────────────────────────── + if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then + echo "" + echo "━━━ $ICON_ERROR Unexpected Exits ━━━" + CRASHED=$(docker ps -a \ + --filter status=exited \ + --format "{{.Names}}|{{.Status}}" 2>/dev/null | \ + grep -v "Exited (0)") + + if [[ -z "$CRASHED" ]]; then + success "No unexpected exits" + else + while IFS='|' read -r container status; do + [[ -z "$container" ]] && continue + [[ -n "${IGNORE_MAP[$container]:-}" ]] && \ + { info "Skipping $container (ignore list)"; continue; } + is_skipped "$container" && \ + { warn "$container — crashed but on skip list, manual intervention needed"; continue; } + + # Skip Tier 1 required containers — already handled above + SKIP=false + for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do + [[ "$container" == "$req" ]] && SKIP=true && break + done + [[ "$SKIP" == true ]] && continue + + error "$container — $status (unexpected exit)" + ((T2_WARNINGS++)) + result=0 + safe_restart "$container" "unexpected exit" || result=$? + if [[ $result -eq 0 ]]; then + ((T2_RESTARTS++)) + queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning" + elif [[ $result -eq 2 ]]; then + : # skip list message already queued + fi + done <<< "$CRASHED" + fi + fi +fi + +# ----------------------------------------------------------------------------------------------- +# Send batched notifications +# ----------------------------------------------------------------------------------------------- +flush_notify # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Summary ━━━ # ----------------------------------------------------------------------------------------------- +TOTAL_RESTARTS=$(( T1_RESTARTS + T2_RESTARTS )) +TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS )) + echo "" -echo "━━━━━ $ICON_SUMMARY WATCHDOG SUMMARY ━━━━━" -echo "$ICON_TIME $(date '+%Y-%m-%d %H:%M:%S')" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -echo "$ICON_WATCHDOG Monitored: ${#WATCHDOG_CONTAINERS[@]} containers" -echo "$ICON_CONTAINERS Required: ${#WATCHDOG_REQUIRED_CONTAINERS[@]} containers" -[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]}" -[[ "$DRY_RUN" == true ]] && echo "$ICON_WARN Dry Run: no actions taken" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +echo "━━━━━ $ICON_SUMMARY DOCKER WATCHDOG SUMMARY ━━━━━" +echo "$ICON_WATCHDOG Tier 1 (strict): $T1_RESTARTS restarts / $T1_WARNINGS warnings" +echo "$ICON_WATCHDOG Tier 2 (global): $T2_RESTARTS restarts / $T2_WARNINGS warnings" +echo "$ICON_WATCHDOG Total: $TOTAL_RESTARTS restarts / $TOTAL_WARNINGS warnings" +[[ "$IN_GRACE_PERIOD" == "true" ]] && \ + echo "$ICON_WARN Grace period: active — restarts suppressed" +echo "" +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no changes made" +elif [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then + echo "$ICON_WARN Status: $TOTAL_RESTARTS container(s) restarted / $TOTAL_WARNINGS warning(s)" +else + echo "$ICON_DONE Status: $ICON_SUCCESS ALL CONTAINERS HEALTHY" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Master.conf b/Master.conf index 1759b25..2926a25 100644 --- a/Master.conf +++ b/Master.conf @@ -380,33 +380,36 @@ WEEKLY_RESTART_CONTAINERS=( ) # ━━━ Docker Watchdog ━━━ -# First line of defense for container health — runs every 15 minutes via cron. -# Monitors memory usage, CPU usage and HTTP responsiveness per container. -# Uses a strike system to avoid restarting on brief spikes — sustained issues trigger restart. -# Works alongside system_watchdog.sh — containers first, system reboot is the last resort. - -# Containers to monitor with their memory hard limits in MB. -# Memory hard limit exceeded → immediate restart (no strike system for memory). -# CPU and HTTP use strike system — see CPU_FAIL_LIMIT and RESP_FAIL_LIMIT below. +# Two-tier self-healing container monitoring: +# Tier 1 — strict monitoring of explicitly configured containers +# Tier 2 — global health scan of ALL running containers +# +# Cross-cutting intelligence applies to both tiers: +# Startup grace — skip restarts while system is still booting +# Dependency order — restart database before app, not the other way around +# Restart loop — stop restarting after limit hit → skip list → notify critical +# Skip list — persistent across reboots, auto-clears when container recovers +# Batch notify — one clean summary per run instead of one ping per event + +# ── Tier 1 — Strict Monitoring ──────────────────────────────────────────────────────────── + +# Memory hard limits in MB — immediate restart if exceeded, no strike system # 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240 # 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024 declare -A WATCHDOG_CONTAINERS=( - ["Emby"]=16384 # 16GB — media server, transcoding can spike high - ["LidaTube"]=6144 # 6GB — YouTube downloader - ["Tdarr"]=6144 # 6GB — transcoding node - ["Code-Server"]=1024 # 1GB — VS Code server + ["Emby"]=16384 + ["LidaTube"]=6144 + ["Tdarr"]=6144 + ["Code-Server"]=1024 ) - -# Containers to check HTTP responsiveness via curl — omit a container to skip its HTTP check. -# curl checks the URL and considers the container unresponsive if it times out or errors. + +# HTTP responsiveness checks — omit container to skip its HTTP check declare -A WATCHDOG_CONTAINER_URLS=( ["Emby"]="http://localhost:8096" ) - -# Containers that should always be running — monitored for unexpected stops. -# Strike system used — tries restart on each strike up to SYS_WATCHDOG_STRIKE_LIMIT. -# If restart fails after strike limit → added to persistent skip list on /boot/ -# Skip list auto-clears when container recovers after reboot or manual fix. + +# Containers that must always be running — strike system, persistent skip list on /boot/ +# Skip list auto-clears when container recovers — no manual intervention for normal recovery WATCHDOG_REQUIRED_CONTAINERS=( "NginxProxyManager" "Lldap-Gmer4Lfe" @@ -416,23 +419,74 @@ WATCHDOG_REQUIRED_CONTAINERS=( "Authelia-Secondary" "Redis-Authelia-Secondary" ) - -# Strike state file — /tmp resets on reboot which is correct behaviour for strike tracking + +# Strike thresholds WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" - -# CPU thresholds — normalised against total core count automatically at runtime. -# A container using 85% of one core on a 16-core system = ~5.3% normalised. - SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU - HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU - CPU_FAIL_LIMIT=2 # consecutive strikes before container restart - -# Memory soft threshold — warn when container reaches this % of its hard limit. -# Hard limit exceeded triggers immediate restart regardless of strikes. - SOFT_MEM_THRESHOLD=80 - -# HTTP responsiveness check settings - RESP_FAIL_LIMIT=2 # consecutive failed curl checks before restart - CURL_TIMEOUT=5 # seconds before curl gives up per check + SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU + HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU + CPU_FAIL_LIMIT=2 # consecutive CPU strikes before restart + SOFT_MEM_THRESHOLD=80 # warn when container reaches this % of hard limit + RESP_FAIL_LIMIT=2 # consecutive failed HTTP checks before restart + CURL_TIMEOUT=5 # seconds before curl gives up per check + +# ── Tier 2 — Global Health Scan ─────────────────────────────────────────────────────────── + +# Master toggle — false disables Tier 2 entirely + WATCHDOG_SCAN_ALL=true + +# Containers to skip in Tier 2 — add containers expected to be in a non-running state +# or managed by other systems that should not be auto-restarted +WATCHDOG_SCAN_IGNORE=( + # "container-name" +) + +# Individual Tier 2 check toggles — disable checks that cause false positives + WATCHDOG_RESTART_UNHEALTHY=true # restart containers with unhealthy Docker health status + WATCHDOG_RESTART_DEAD=true # remove and restart containers in dead state + WATCHDOG_RESTART_CRASHED=true # restart containers that exited with non-zero exit code + WATCHDOG_NOTIFY_OOM=true # restart and notify when OOM killed by kernel + WATCHDOG_NOTIFY_CRASHLOOP=true # notify when Docker restart count is climbing + +# Crash loop threshold — notify critical if Docker has restarted container this many times + WATCHDOG_CRASH_LIMIT=5 + +# ── Cross-cutting Intelligence ──────────────────────────────────────────────────────────── + +# Startup grace period — skip restarts while system is still booting +# Prevents false positives while containers are coming up after array start + WATCHDOG_STARTUP_GRACE=300 # seconds after boot before watchdog acts on failures + +# Restart loop protection — stops hammering broken containers +# Tracks watchdog-initiated restarts per container in a bounded /boot/ file +# After limit hit → container added to skip list → notify critical → manual intervention +# Skip list auto-clears when container is found running again + WATCHDOG_CONTAINER_RESTART_LIMIT=3 # max watchdog restarts allowed in window + WATCHDOG_CONTAINER_RESTART_WINDOW=1 # hours — rolling window for restart count + WATCHDOG_CONTAINER_RESTART_LOG="/boot/config/container_restart_history.db" + # /boot/ survives reboots — bounded, auto-purges old entries + +# Dependency ordering — skip restarting a container if its dependency is also down +# Dependency gets restarted first, dependent picked up on the next watchdog cycle +# Format: ["dependent"]="dependency1 dependency2" +declare -A WATCHDOG_DEPENDENCIES=( + ["Authelia"]="Mariadb-Authelia Redis-Authelia" + ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary" + ["NextCloud"]="Postgres-NextCloud" +) + +# Notification batching — one clean summary per run instead of one ping per event +# true = batch all events into a single notification at end of run +# false = send individual notification per event as it happens + WATCHDOG_BATCH_NOTIFY=true + +# ━━━ Docker Network Connect ━━━ +NETWORK_CONNECT_CONTAINERS=( + "memcached" + "Npm-CrowdSec" +) +NETWORK_CONNECT_NETWORKS=( + "nextcloud-aio" +) # ━━━ Docker Network Connect ━━━ # Connects containers to extra Docker networks on array start.