#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Docker Watchdog --------------------------------------------
# -----------------------------------------------------------------------------------------------
# Two-tier self-healing container monitoring system.
#
# Tier 1 — Strict monitoring (configured containers only)
#   Memory hard limits    — immediate restart if exceeded
#   CPU thresholds        — strike system, restart after CPU_FAIL_LIMIT strikes
#   HTTP responsiveness   — strike system, restart after RESP_FAIL_LIMIT strikes
#   Required containers   — must always be running, strike system with skip list
#
# Tier 2 — Global health scan (all running containers)
#   Unhealthy status      — Docker HEALTHCHECK unhealthy → restart
#   OOM killed            — kernel killed container → restart + notify
#   Crash loop detection  — RestartCount climbing → notify, critical above limit
#   Dead containers       — remove and restart
#   Unexpected exits      — non-zero exit code → restart
#
# Cross-cutting intelligence (applies to both tiers):
#   Startup grace period  — skip restarts while system is still booting
#   Dependency ordering   — restart database before app, not the other way around
#   Restart loop protect  — stop restarting after X restarts in X hours → skip list
#   Skip list auto-clear  — clears when container recovers, fresh start
#   Notification batching — one clean summary per run, not one ping per event
#
# State files:
#   WATCHDOG_STATE_FILE              — strike counts (/tmp — resets on reboot)
#   SYS_WATCHDOG_FAILED_FILE         — persistent skip list (/boot — survives reboots)
#   WATCHDOG_CONTAINER_RESTART_LOG   — restart history for loop detection (/boot)
#
# All configuration in Master.conf under Docker Watchdog section.
# Supports --dry-run and --status.
# -----------------------------------------------------------------------------------------------

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"

parse_args "$@"

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

success "Running as root"

if ! command -v docker >/dev/null 2>&1; then
    error "Docker not found"
    exit 1
fi

success "Docker found"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"

# State files
touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \
      "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null

# Build ignore map for Tier 2
declare -A IGNORE_MAP
for c in "${WATCHDOG_SCAN_IGNORE[@]}"; do
    [[ -n "$c" ]] && IGNORE_MAP["$c"]=1
done

# Batch notification collector
NOTIFY_EVENTS=()

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_CONTAINERS Watched:          ${!WATCHDOG_CONTAINERS[@]}"
    echo "$ICON_CONTAINERS Required:         ${WATCHDOG_REQUIRED_CONTAINERS[*]}"
    echo "$ICON_WATCHDOG   Scan all:         $WATCHDOG_SCAN_ALL"
    echo "$ICON_WATCHDOG   Startup grace:    ${WATCHDOG_STARTUP_GRACE}s"
    echo "$ICON_WATCHDOG   Restart limit:    $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
    echo "$ICON_WATCHDOG   Batch notify:     $WATCHDOG_BATCH_NOTIFY"
    echo "$ICON_WATCHDOG   Ignore list:      ${WATCHDOG_SCAN_IGNORE[*]:-none}"
    echo "$ICON_GEAR       Dry Run:          $DRY_RUN"
    UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
    echo "$ICON_TIME        System uptime:    $(format_duration $UPTIME_S)"
    [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]] && \
        warn "Within startup grace period — restarts suppressed"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# -----------------------------------------------------------------------------------------------
# HELPERS
# -----------------------------------------------------------------------------------------------

# Strike count management
get_strikes() {
    grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"
}

set_strikes() {
    local container="$1" count="$2" file="$3"
    if grep -q "^${container}:" "$file" 2>/dev/null; then
        sed -i "s/^${container}:.*/${container}:${count}/" "$file"
    else
        echo "${container}:${count}" >> "$file"
    fi
}

# Persistent skip list management
is_skipped() {
    grep -q "^${1}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
}

add_to_skip_list() {
    local container="$1" reason="$2"
    if ! is_skipped "$container"; then
        echo "$container" >> "$SYS_WATCHDOG_FAILED_FILE"
        error "$container added to skip list — $reason"
        queue_notify "$container added to skip list on $(hostname) — $reason — manual intervention needed" "critical"
    fi
}

remove_from_skip_list() {
    sed -i "/^${1}$/d" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
    success "$1 removed from skip list — recovered"
}

# Restart loop tracking — /boot/ bounded file, auto-purges old entries
log_restart() {
    local container="$1"
    local now
    now=$(date '+%Y-%m-%d %H:%M:%S')
    local cutoff
    cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S')

    # Append this restart
    echo "${container}|${now}" >> "$WATCHDOG_CONTAINER_RESTART_LOG"

    # Purge entries older than window — keeps file bounded
    local tmp="${WATCHDOG_CONTAINER_RESTART_LOG}.tmp"
    awk -F'|' -v cutoff="$cutoff" '$2 >= cutoff' \
        "$WATCHDOG_CONTAINER_RESTART_LOG" > "$tmp" && \
        mv "$tmp" "$WATCHDOG_CONTAINER_RESTART_LOG"
}

# Count restarts for a container within the current window
get_restart_count() {
    local container="$1"
    local cutoff
    cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S')
    awk -F'|' -v c="$container" -v cutoff="$cutoff" \
        '$1==c && $2>=cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l
}

# Dependency check — returns 0 if all dependencies are running
dependencies_satisfied() {
    local container="$1"
    local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
    [[ -z "$deps" ]] && return 0

    for dep in $deps; do
        local status
        status=$(docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null)
        if [[ "$status" != "true" ]]; then
            warn "$container — dependency $dep is not running — skipping restart this cycle"
            return 1
        fi
    done
    return 0
}

# Restart a container with restart loop protection
# Returns 0 on success, 1 on failure, 2 if loop limit hit
safe_restart() {
    local container="$1" reason="$2"

    # Check restart loop
    local restart_count
    restart_count=$(get_restart_count "$container")

    if [[ "$restart_count" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then
        add_to_skip_list "$container" \
            "restarted $restart_count times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
        return 2
    fi

    # Check dependencies
    dependencies_satisfied "$container" || return 1

    # Check startup grace
    local uptime_s
    uptime_s=$(awk '{print int($1)}' /proc/uptime)
    if [[ "$uptime_s" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
        warn "$container — within startup grace period (${uptime_s}s / ${WATCHDOG_STARTUP_GRACE}s) — skipping restart"
        return 1
    fi

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would restart $container ($reason)"
        return 0
    fi

    info "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]..."
    if docker restart "$container" >/dev/null 2>&1; then
        success "$ICON_STARTED $container restarted"
        log_restart "$container"
        return 0
    else
        error "Failed to restart $container"
        return 1
    fi
}

# Queue a notification event for batching
queue_notify() {
    local message="$1" severity="${2:-warning}"
    NOTIFY_EVENTS+=("${severity}|${message}")
    log "Queued: $message"
}

# Send all queued notifications
flush_notify() {
    [[ ${#NOTIFY_EVENTS[@]} -eq 0 ]] && return

    if [[ "$WATCHDOG_BATCH_NOTIFY" == "true" ]]; then
        # Build single batched message
        local critical_count=0 warning_count=0
        local messages=()
        local highest_severity="normal"

        for event in "${NOTIFY_EVENTS[@]}"; do
            local sev="${event%%|*}"
            local msg="${event#*|}"
            messages+=("$msg")
            [[ "$sev" == "critical" ]] && ((critical_count++)) && highest_severity="warning"
            [[ "$sev" == "warning" ]] && ((warning_count++)) && \
                [[ "$highest_severity" == "normal" ]] && highest_severity="warning"
        done

        local summary
        summary=$(printf '%s. ' "${messages[@]}")
        notify "Docker Watchdog on $(hostname) — ${#NOTIFY_EVENTS[@]} event(s): $summary" \
            "Docker Watchdog" "$highest_severity"
    else
        # Send individual notifications
        for event in "${NOTIFY_EVENTS[@]}"; do
            local sev="${event%%|*}"
            local msg="${event#*|}"
            [[ "$sev" == "critical" ]] && sev="warning"
            notify "$msg" "Docker Watchdog" "$sev"
        done
    fi

    NOTIFY_EVENTS=()
}

# -----------------------------------------------------------------------------------------------
# STARTUP GRACE CHECK
# -----------------------------------------------------------------------------------------------
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
IN_GRACE_PERIOD=false

if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
    GRACE_REMAINING=$(( WATCHDOG_STARTUP_GRACE - UPTIME_SECONDS ))
    warn "System uptime $(format_duration $UPTIME_SECONDS) — startup grace period active (${GRACE_REMAINING}s remaining)"
    warn "Checks will run but restarts are suppressed during grace period"
    IN_GRACE_PERIOD=true
else
    info "System uptime $(format_duration $UPTIME_SECONDS) — grace period passed"
fi

# -----------------------------------------------------------------------------------------------
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# TIER 1 — Strict Monitoring
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# -----------------------------------------------------------------------------------------------

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  $ICON_WATCHDOG TIER 1 — Strict Monitoring"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

T1_RESTARTS=0
T1_WARNINGS=0

# ── Required Containers ──────────────────────────────────────────────────────────────────────
if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_CONTAINERS Required Containers ━━━"

    for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
        [[ -z "$container" ]] && continue

        STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)

        # Check skip list
        if is_skipped "$container"; then
            if [[ "$STATUS" == "true" ]]; then
                remove_from_skip_list "$container"
                set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
                # Clear restart history on recovery
                sed -i "/^${container}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
                queue_notify "$container recovered on $(hostname) — removed from skip list" "normal"
            else
                warn "$ICON_NOT_RUNNING $container — on skip list, manual intervention needed"
                RESTART_COUNT=$(get_restart_count "$container")
                warn "  Restarted $RESTART_COUNT time(s) in last ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
            fi
            continue
        fi

        if [[ "$STATUS" == "true" ]]; then
            success "$ICON_RUNNING $container — running"
            set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
        else
            STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
            STRIKES=$(( STRIKES + 1 ))
            set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
            warn "$ICON_NOT_RUNNING $container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
            ((T1_WARNINGS++))

            if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
                result=0
                safe_restart "$container" "required container down" || result=$?
                case $result in
                    0)
                        set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
                        ((T1_RESTARTS++))
                        queue_notify "$container was down and restarted on $(hostname)" "warning"
                        ;;
                    2)
                        : # already added to skip list in safe_restart
                        ;;
                    *)
                        queue_notify "$container failed to restart on $(hostname) — check dependencies and logs" "warning"
                        ;;
                esac
            fi
        fi
    done
fi

# ── Memory and CPU Monitoring ────────────────────────────────────────────────────────────────
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_MEM Memory / CPU Monitoring ━━━"

    STATS=$(docker stats --no-stream \
        --format "{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}" 2>/dev/null)
    TOTAL_CORES=$(nproc 2>/dev/null || echo 1)

    for container in "${!WATCHDOG_CONTAINERS[@]}"; do
        MEM_LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
        CONTAINER_STATS=$(echo "$STATS" | grep "^${container}|" | head -1)
        [[ -z "$CONTAINER_STATS" ]] && continue

        MEM_USAGE=$(echo "$CONTAINER_STATS" | cut -d'|' -f2 | awk '{print $1}')
        MEM_UNIT=$(echo "$MEM_USAGE" | grep -oE '[A-Za-z]+')
        MEM_VALUE=$(echo "$MEM_USAGE" | grep -oE '[0-9.]+')

        case "$MEM_UNIT" in
            GiB|GB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE * 1024}") ;;
            MiB|MB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE}") ;;
            KiB|KB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE / 1024}") ;;
            *)       MEM_MB=0 ;;
        esac

        MEM_PCT=$(awk "BEGIN {printf \"%.0f\", $MEM_MB * 100 / $MEM_LIMIT_MB}")
        CPU_RAW=$(echo "$CONTAINER_STATS" | cut -d'|' -f3 | tr -d '%')
        CPU_NORM=$(awk "BEGIN {printf \"%.1f\", $CPU_RAW / $TOTAL_CORES}")
        CPU_INT=$(printf "%.0f" "$CPU_NORM")

        # Memory check
        if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then
            error "$container — memory ${MEM_VALUE}${MEM_UNIT} exceeded hard limit ${MEM_LIMIT_MB}MB"
            safe_restart "$container" "memory hard limit exceeded"
            ((T1_RESTARTS++))
            queue_notify "$container exceeded memory limit on $(hostname) — ${MEM_VALUE}${MEM_UNIT} / ${MEM_LIMIT_MB}MB — restarted" "warning"
        elif [[ "$MEM_PCT" -ge "$SOFT_MEM_THRESHOLD" ]]; then
            warn "$container — memory ${MEM_VALUE}${MEM_UNIT} (${MEM_PCT}% of ${MEM_LIMIT_MB}MB)"
            ((T1_WARNINGS++))
        else
            success "$container — memory ${MEM_VALUE}${MEM_UNIT} (${MEM_PCT}%) CPU ${CPU_NORM}%"
        fi

        # CPU check
        if [[ "$CPU_INT" -ge "$HARD_CPU_THRESHOLD" ]]; then
            CPU_STRIKES=$(get_strikes "${container}_cpu" "$WATCHDOG_STATE_FILE")
            CPU_STRIKES=$(( CPU_STRIKES + 1 ))
            set_strikes "${container}_cpu" "$CPU_STRIKES" "$WATCHDOG_STATE_FILE"
            warn "$container — CPU ${CPU_NORM}% (strike $CPU_STRIKES/$CPU_FAIL_LIMIT)"
            if [[ "$CPU_STRIKES" -ge "$CPU_FAIL_LIMIT" ]]; then
                safe_restart "$container" "CPU threshold exceeded"
                set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
                ((T1_RESTARTS++))
                queue_notify "$container CPU ${CPU_NORM}% on $(hostname) — restarted after $CPU_FAIL_LIMIT strikes" "warning"
            fi
        else
            set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
        fi
    done
fi

# ── HTTP Responsiveness ──────────────────────────────────────────────────────────────────────
if [[ ${#WATCHDOG_CONTAINER_URLS[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_NET HTTP Responsiveness ━━━"

    for container in "${!WATCHDOG_CONTAINER_URLS[@]}"; do
        URL="${WATCHDOG_CONTAINER_URLS[$container]}"
        if curl -sf --max-time "$CURL_TIMEOUT" "$URL" >/dev/null 2>&1; then
            success "$container — responding at $URL"
            set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
        else
            HTTP_STRIKES=$(get_strikes "${container}_http" "$WATCHDOG_STATE_FILE")
            HTTP_STRIKES=$(( HTTP_STRIKES + 1 ))
            set_strikes "${container}_http" "$HTTP_STRIKES" "$WATCHDOG_STATE_FILE"
            warn "$container — not responding at $URL (strike $HTTP_STRIKES/$RESP_FAIL_LIMIT)"
            ((T1_WARNINGS++))
            if [[ "$HTTP_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then
                safe_restart "$container" "HTTP unresponsive"
                set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
                ((T1_RESTARTS++))
                queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning"
            fi
        fi
    done
fi

# -----------------------------------------------------------------------------------------------
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# TIER 2 — Global Health Scan
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# -----------------------------------------------------------------------------------------------

T2_RESTARTS=0
T2_WARNINGS=0

if [[ "$WATCHDOG_SCAN_ALL" != "true" ]]; then
    info "Global scan disabled (WATCHDOG_SCAN_ALL=false) — skipping Tier 2"
else
    echo ""
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    echo "  $ICON_WATCHDOG TIER 2 — Global Health Scan"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

    ALL_CONTAINERS=$(docker ps --format "{{.Names}}" 2>/dev/null)

    # ── Unhealthy containers ─────────────────────────────────────────────────────────────────
    if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then
        echo ""
        echo "━━━ $ICON_HEALTH Unhealthy Status ━━━"
        UNHEALTHY=$(docker ps --filter health=unhealthy --format "{{.Names}}" 2>/dev/null)

        if [[ -z "$UNHEALTHY" ]]; then
            success "No unhealthy containers"
        else
            while IFS= read -r container; do
                [[ -z "$container" ]] && continue
                [[ -n "${IGNORE_MAP[$container]:-}" ]] && \
                    { info "Skipping $container (ignore list)"; continue; }
                is_skipped "$container" && \
                    { warn "$container — unhealthy but on skip list, manual intervention needed"; continue; }

                error "$container — Docker health status: unhealthy"
                ((T2_WARNINGS++))
                result=0
                safe_restart "$container" "unhealthy health status" || result=$?
                if [[ $result -eq 0 ]]; then
                    ((T2_RESTARTS++))
                    queue_notify "$container unhealthy on $(hostname) — restarted" "warning"
                elif [[ $result -eq 2 ]]; then
                    : # skip list message already queued
                fi
            done <<< "$UNHEALTHY"
        fi
    fi

    # ── OOM killed containers ────────────────────────────────────────────────────────────────
    if [[ "$WATCHDOG_NOTIFY_OOM" == "true" ]]; then
        echo ""
        echo "━━━ $ICON_MEM OOM Kill Check ━━━"
        OOM_FOUND=false

        while IFS= read -r container; do
            [[ -z "$container" ]] && continue
            [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
            is_skipped "$container" && continue

            OOM=$(docker inspect -f '{{.State.OOMKilled}}' "$container" 2>/dev/null)
            if [[ "$OOM" == "true" ]]; then
                error "$container — OOM killed by kernel"
                OOM_FOUND=true
                ((T2_WARNINGS++))
                result=0
                safe_restart "$container" "OOM killed" || result=$?
                if [[ $result -eq 0 ]]; then
                    ((T2_RESTARTS++))
                    queue_notify "$container OOM killed on $(hostname) — system under memory pressure — restarted" "warning"
                fi
            fi
        done <<< "$ALL_CONTAINERS"

        [[ "$OOM_FOUND" == false ]] && success "No OOM kills detected"
    fi

    # ── Crash loop detection ─────────────────────────────────────────────────────────────────
    if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then
        echo ""
        echo "━━━ $ICON_REBOOT_SMART Crash Loop Detection ━━━"
        CRASH_FOUND=false

        while IFS= read -r container; do
            [[ -z "$container" ]] && continue
            [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
            is_skipped "$container" && continue

            RESTART_COUNT=$(docker inspect -f '{{.RestartCount}}' "$container" 2>/dev/null || echo 0)
            PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d: -f2 || echo 0)

            # Update baseline
            set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE"

            if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then
                CRASH_FOUND=true
                ((T2_WARNINGS++))
                if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then
                    error "$container — crash loop CRITICAL: $RESTART_COUNT total Docker restarts"
                    queue_notify "$container crash loop CRITICAL on $(hostname) — $RESTART_COUNT Docker restarts — manual intervention needed" "critical"
                else
                    warn "$container — Docker restarted since last check (total: $RESTART_COUNT)"
                    queue_notify "$container restarted on $(hostname) — Docker restart count: $RESTART_COUNT" "warning"
                fi
            elif [[ "$RESTART_COUNT" -gt 0 ]]; then
                log "$container — Docker restart count: $RESTART_COUNT (stable)"
            fi
        done <<< "$ALL_CONTAINERS"

        [[ "$CRASH_FOUND" == false ]] && success "No crash loops detected"
    fi

    # ── Dead containers ──────────────────────────────────────────────────────────────────────
    if [[ "$WATCHDOG_RESTART_DEAD" == "true" ]]; then
        echo ""
        echo "━━━ $ICON_STOPPED Dead Containers ━━━"
        DEAD=$(docker ps -a --filter status=dead --format "{{.Names}}" 2>/dev/null)

        if [[ -z "$DEAD" ]]; then
            success "No dead containers"
        else
            while IFS= read -r container; do
                [[ -z "$container" ]] && continue
                [[ -n "${IGNORE_MAP[$container]:-}" ]] && \
                    { info "Skipping $container (ignore list)"; continue; }
                is_skipped "$container" && \
                    { warn "$container — dead but on skip list, manual intervention needed"; continue; }

                error "$container — status: dead"
                ((T2_WARNINGS++))
                RESTART_COUNT=$(get_restart_count "$container")
                if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then
                    add_to_skip_list "$container" "dead state — restarted $RESTART_COUNT times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
                elif [[ "$DRY_RUN" == false ]]; then
                    docker rm "$container" >/dev/null 2>&1
                    if docker start "$container" >/dev/null 2>&1; then
                        success "$container removed from dead state and restarted"
                        log_restart "$container"
                        ((T2_RESTARTS++))
                        queue_notify "$container was dead on $(hostname) — removed and restarted" "warning"
                    else
                        error "$container failed to restart after dead state"
                        queue_notify "$container dead and failed to restart on $(hostname) — manual intervention needed" "warning"
                    fi
                else
                    warn "DRY RUN — would remove and restart $container"
                fi
            done <<< "$DEAD"
        fi
    fi

    # ── Unexpected exits ─────────────────────────────────────────────────────────────────────
    if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then
        echo ""
        echo "━━━ $ICON_ERROR Unexpected Exits ━━━"
        CRASHED=$(docker ps -a \
            --filter status=exited \
            --format "{{.Names}}|{{.Status}}" 2>/dev/null | \
            grep -v "Exited (0)")

        if [[ -z "$CRASHED" ]]; then
            success "No unexpected exits"
        else
            while IFS='|' read -r container status; do
                [[ -z "$container" ]] && continue
                [[ -n "${IGNORE_MAP[$container]:-}" ]] && \
                    { info "Skipping $container (ignore list)"; continue; }
                is_skipped "$container" && \
                    { warn "$container — crashed but on skip list, manual intervention needed"; continue; }

                # Skip Tier 1 required containers — already handled above
                SKIP=false
                for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
                    [[ "$container" == "$req" ]] && SKIP=true && break
                done
                [[ "$SKIP" == true ]] && continue

                error "$container — $status (unexpected exit)"
                ((T2_WARNINGS++))
                result=0
                safe_restart "$container" "unexpected exit" || result=$?
                if [[ $result -eq 0 ]]; then
                    ((T2_RESTARTS++))
                    queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning"
                elif [[ $result -eq 2 ]]; then
                    : # skip list message already queued
                fi
            done <<< "$CRASHED"
        fi
    fi
fi

# -----------------------------------------------------------------------------------------------
# Send batched notifications
# -----------------------------------------------------------------------------------------------
flush_notify

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
TOTAL_RESTARTS=$(( T1_RESTARTS + T2_RESTARTS ))
TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS ))

echo ""
echo "━━━━━ $ICON_SUMMARY DOCKER WATCHDOG SUMMARY ━━━━━"
echo "$ICON_WATCHDOG Tier 1 (strict):   $T1_RESTARTS restarts / $T1_WARNINGS warnings"
echo "$ICON_WATCHDOG Tier 2 (global):   $T2_RESTARTS restarts / $T2_WARNINGS warnings"
echo "$ICON_WATCHDOG Total:             $TOTAL_RESTARTS restarts / $TOTAL_WARNINGS warnings"
[[ "$IN_GRACE_PERIOD" == "true" ]] && \
    echo "$ICON_WARN Grace period:      active — restarts suppressed"
echo ""
if [[ "$DRY_RUN" == true ]]; then
    echo "$ICON_WARN Status: DRY RUN — no changes made"
elif [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then
    echo "$ICON_WARN Status: $TOTAL_RESTARTS container(s) restarted / $TOTAL_WARNINGS warning(s)"
else
    echo "$ICON_DONE Status: $ICON_SUCCESS ALL CONTAINERS HEALTHY"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"