#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Docker Watchdog --------------------------------------------
# -----------------------------------------------------------------------------------------------
# Two-tier self-healing container monitoring system — runs continuously as a background process.
# Started by array_start.sh at array start — runs until array stops or SIGTERM received.
#
# Tier 1 — Strict monitoring (configured containers only)
#   Memory hard limits    — immediate restart if exceeded
#   CPU thresholds        — strike system, restart after CPU_FAIL_LIMIT strikes
#   HTTP responsiveness   — strike system, restart after RESP_FAIL_LIMIT strikes
#   Required containers   — must always be running, strike system with skip list
#
# Tier 2 — Global health scan (all running containers)
#   Unhealthy status      — Docker HEALTHCHECK unhealthy → restart
#   OOM killed            — kernel killed container → restart + notify
#   Crash loop detection  — RestartCount climbing → notify, critical above limit
#   Dead containers       — remove and restart
#   Unexpected exits      — non-zero exit code → restart
#
# Cross-cutting intelligence:
#   Startup grace period  — skip restarts while system is still booting
#   Dependency ordering   — restart database before app
#   Restart loop protect  — stop restarting after X restarts in X hours → skip list
#   Skip list auto-clear  — clears when container recovers
#   Notification batching — one clean summary per cycle, not one ping per event
#   Quiet when healthy    — only logs when something needs attention
#   Parity awareness      — skips restarts during parity check
#
# Continuous loop:
#   Checks run every DOCKER_WATCHDOG_INTERVAL seconds (default 900 = 15min)
#   Clean shutdown on SIGTERM/SIGINT — sent by array stop
#   Variables scoped per-cycle — no state accumulation between cycles
#
# State files:
#   WATCHDOG_STATE_FILE              — strike counts (/tmp — resets on reboot)
#   SYS_WATCHDOG_FAILED_FILE         — persistent skip list (/boot — survives reboots)
#   WATCHDOG_CONTAINER_RESTART_LOG   — restart history for loop detection
#
# All configuration in Master.conf under Docker Watchdog section.
# Supports --dry-run and --status.
# -----------------------------------------------------------------------------------------------

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"

parse_args "$@"

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup — runs once at start ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

success "Running as root"

acquire_lock "continuous"

# Select correct per-host watchdog lists — done once at startup
detect_hosts

if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
    WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}")
    declare -A WATCHDOG_CONTAINER_URLS
    for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do
        WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}"
    done
else
    WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}")
    declare -A WATCHDOG_CONTAINER_URLS
    for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do
        WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}"
    done
fi

info "Watchdog running as: $LOCAL_SERVER_NAME"
info "Check interval: ${DOCKER_WATCHDOG_INTERVAL}s"

if ! command -v docker >/dev/null 2>&1; then
    error "Docker not found"
    exit 1
fi

success "Docker found"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"

# Ensure state files exist
touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \
      "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_CONTAINERS Watched:          ${!WATCHDOG_CONTAINERS[@]}"
    echo "$ICON_CONTAINERS Required:         ${WATCHDOG_REQUIRED_CONTAINERS[*]}"
    echo "$ICON_WATCHDOG   Scan all:         $WATCHDOG_SCAN_ALL"
    echo "$ICON_WATCHDOG   Interval:         ${DOCKER_WATCHDOG_INTERVAL}s"
    echo "$ICON_WATCHDOG   Startup grace:    ${WATCHDOG_STARTUP_GRACE}s"
    echo "$ICON_WATCHDOG   Restart limit:    $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
    echo "$ICON_WATCHDOG   Batch notify:     $WATCHDOG_BATCH_NOTIFY"
    echo "$ICON_WATCHDOG   Ignore list:      ${WATCHDOG_SCAN_IGNORE[*]:-none}"
    echo "$ICON_GEAR       Dry Run:          $DRY_RUN"
    UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
    echo "$ICON_TIME        System uptime:    $(format_duration $UPTIME_S)"
    [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]] && \
        warn "Within startup grace period — restarts suppressed"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# -----------------------------------------------------------------------------------------------
# HELPERS — defined once, used every cycle
# -----------------------------------------------------------------------------------------------

get_strikes() {
    grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"
}

set_strikes() {
    local container="$1" count="$2" file="$3"
    if grep -q "^${container}:" "$file" 2>/dev/null; then
        sed -i "s/^${container}:.*/${container}:${count}/" "$file"
    else
        echo "${container}:${count}" >> "$file"
    fi
}

is_skipped() {
    grep -q "^${1}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
}

add_to_skip_list() {
    local container="$1" reason="$2"
    if ! is_skipped "$container"; then
        echo "$container" >> "$SYS_WATCHDOG_FAILED_FILE"
        error "$container added to skip list — $reason"
        queue_notify "$container added to skip list on $(hostname) — $reason — manual intervention needed" "critical"
    fi
}

remove_from_skip_list() {
    sed -i "/^${1}$/d" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
    success "$1 removed from skip list — recovered"
}

log_restart() {
    local container="$1"
    local now
    now=$(date '+%Y-%m-%d %H:%M:%S')
    local cutoff
    cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S')
    echo "${container}|${now}" >> "$WATCHDOG_CONTAINER_RESTART_LOG"
    local tmp="${WATCHDOG_CONTAINER_RESTART_LOG}.tmp"
    awk -F'|' -v cutoff="$cutoff" '$2 >= cutoff' \
        "$WATCHDOG_CONTAINER_RESTART_LOG" > "$tmp" && \
        mv "$tmp" "$WATCHDOG_CONTAINER_RESTART_LOG"
}

get_restart_count() {
    local container="$1"
    local cutoff
    cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S')
    awk -F'|' -v c="$container" -v cutoff="$cutoff" \
        '$1==c && $2>=cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l
}

dependencies_satisfied() {
    local container="$1"
    local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
    [[ -z "$deps" ]] && return 0
    for dep in $deps; do
        local status
        status=$(docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null)
        if [[ "$status" != "true" ]]; then
            warn "$container — dependency $dep is not running — skipping restart this cycle"
            return 1
        fi
    done
    return 0
}

safe_restart() {
    local container="$1" reason="$2"
    local restart_count
    restart_count=$(get_restart_count "$container")
    if [[ "$restart_count" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then
        add_to_skip_list "$container" \
            "restarted $restart_count times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
        return 2
    fi
    dependencies_satisfied "$container" || return 1
    local uptime_s
    uptime_s=$(awk '{print int($1)}' /proc/uptime)
    if [[ "$uptime_s" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
        warn "$container — within startup grace period — skipping restart"
        return 1
    fi
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would restart $container ($reason)"
        return 0
    fi
    info "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]..."
    if docker restart "$container" >/dev/null 2>&1; then
        success "$ICON_STARTED $container restarted"
        log_restart "$container"
        return 0
    else
        error "Failed to restart $container"
        return 1
    fi
}

queue_notify() {
    local message="$1" severity="${2:-warning}"
    NOTIFY_EVENTS+=("${severity}|${message}")
    log "Queued: $message"
}

flush_notify() {
    [[ ${#NOTIFY_EVENTS[@]} -eq 0 ]] && return
    if [[ "$WATCHDOG_BATCH_NOTIFY" == "true" ]]; then
        local highest_severity="normal"
        local messages=()
        for event in "${NOTIFY_EVENTS[@]}"; do
            local sev="${event%%|*}" msg="${event#*|}"
            messages+=("$msg")
            [[ "$sev" == "critical" ]] && highest_severity="warning"
            [[ "$sev" == "warning" && "$highest_severity" == "normal" ]] && highest_severity="warning"
        done
        local summary
        summary=$(printf '%s. ' "${messages[@]}")
        notify "Docker Watchdog on $(hostname) — ${#NOTIFY_EVENTS[@]} event(s): $summary" \
            "Docker Watchdog" "$highest_severity"
    else
        for event in "${NOTIFY_EVENTS[@]}"; do
            local sev="${event%%|*}" msg="${event#*|}"
            [[ "$sev" == "critical" ]] && sev="warning"
            notify "$msg" "Docker Watchdog" "$sev"
        done
    fi
    NOTIFY_EVENTS=()
}

is_parity_running() {
    grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null
}

# -----------------------------------------------------------------------------------------------
# CLEAN SHUTDOWN — trap SIGTERM/SIGINT from array stop
# -----------------------------------------------------------------------------------------------
WATCHDOG_RUNNING=true

cleanup() {
    echo ""
    info "Docker watchdog received shutdown signal — stopping cleanly"
    WATCHDOG_RUNNING=false
    exit 0
}

trap cleanup SIGTERM SIGINT

# -----------------------------------------------------------------------------------------------
# ━━━ CONTINUOUS MONITORING LOOP ━━━
# -----------------------------------------------------------------------------------------------
info "Docker watchdog started — checking every ${DOCKER_WATCHDOG_INTERVAL}s"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

CYCLE=0

while [[ "$WATCHDOG_RUNNING" == true ]]; do
    ((CYCLE++))
    CYCLE_START=$(date +%s)

    # Re-source Master.conf each cycle — picks up any config changes without restart
    source "$SCRIPT_DIR/../Master.conf"

    # Rebuild per-host lists after re-source in case they changed
    if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
        WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}")
        for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do
            WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}"
        done
    else
        WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}")
        for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do
            WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}"
        done
    fi

    # Per-cycle variables — cleared each iteration, no accumulation
    NOTIFY_EVENTS=()
    T1_RESTARTS=0
    T1_WARNINGS=0
    T2_RESTARTS=0
    T2_WARNINGS=0

    # Rebuild ignore map each cycle in case config was updated
    declare -A IGNORE_MAP
    for c in "${WATCHDOG_SCAN_IGNORE[@]}"; do
        [[ -n "$c" ]] && IGNORE_MAP["$c"]=1
    done

    # Startup grace check
    UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
    IN_GRACE_PERIOD=false
    if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
        IN_GRACE_PERIOD=true
    fi

    # Parity check — skip restarts during parity to avoid I/O interference
    if is_parity_running; then
        log "Parity check in progress — skipping restart actions this cycle"
        sleep "$DOCKER_WATCHDOG_INTERVAL"
        continue
    fi

    # ── TIER 1 — Strict Monitoring ──────────────────────────────────────────────────────────

    # Required containers
    if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
        for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
            [[ -z "$container" ]] && continue
            STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)

            if is_skipped "$container"; then
                if [[ "$STATUS" == "true" ]]; then
                    remove_from_skip_list "$container"
                    set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
                    sed -i "/^${container}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
                    queue_notify "$container recovered on $(hostname) — removed from skip list" "normal"
                else
                    warn "$container — on skip list, manual intervention needed"
                fi
                continue
            fi

            if [[ "$STATUS" == "true" ]]; then
                set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
            else
                STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
                STRIKES=$(( STRIKES + 1 ))
                set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
                warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
                ((T1_WARNINGS++))

                if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
                    result=0
                    safe_restart "$container" "required container down" || result=$?
                    case $result in
                        0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
                           ((T1_RESTARTS++))
                           queue_notify "$container was down and restarted on $(hostname)" "warning" ;;
                        2) : ;;
                        *) queue_notify "$container failed to restart on $(hostname)" "warning" ;;
                    esac
                fi
            fi
        done
    fi

    # Memory and CPU monitoring
    if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
        STATS=$(docker stats --no-stream \
            --format "{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}" 2>/dev/null)
        TOTAL_CORES=$(nproc 2>/dev/null || echo 1)

        for container in "${!WATCHDOG_CONTAINERS[@]}"; do
            MEM_LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
            CONTAINER_STATS=$(echo "$STATS" | grep "^${container}|" | head -1)
            [[ -z "$CONTAINER_STATS" ]] && continue

            MEM_USAGE=$(echo "$CONTAINER_STATS" | cut -d'|' -f2 | awk '{print $1}')
            MEM_UNIT=$(echo "$MEM_USAGE" | grep -oE '[A-Za-z]+')
            MEM_VALUE=$(echo "$MEM_USAGE" | grep -oE '[0-9.]+')

            case "$MEM_UNIT" in
                GiB|GB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE * 1024}") ;;
                MiB|MB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE}") ;;
                KiB|KB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE / 1024}") ;;
                *)       MEM_MB=0 ;;
            esac

            CPU_RAW=$(echo "$CONTAINER_STATS" | cut -d'|' -f3 | tr -d '%')
            CPU_NORM=$(awk "BEGIN {printf \"%.1f\", $CPU_RAW / $TOTAL_CORES}")
            CPU_INT=$(printf "%.0f" "$CPU_NORM")

            if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then
                error "$container — memory exceeded hard limit ${MEM_LIMIT_MB}MB"
                safe_restart "$container" "memory hard limit exceeded"
                ((T1_RESTARTS++))
                queue_notify "$container exceeded memory limit on $(hostname) — restarted" "warning"
            fi

            if [[ "$CPU_INT" -ge "$HARD_CPU_THRESHOLD" ]]; then
                CPU_STRIKES=$(get_strikes "${container}_cpu" "$WATCHDOG_STATE_FILE")
                CPU_STRIKES=$(( CPU_STRIKES + 1 ))
                set_strikes "${container}_cpu" "$CPU_STRIKES" "$WATCHDOG_STATE_FILE"
                if [[ "$CPU_STRIKES" -ge "$CPU_FAIL_LIMIT" ]]; then
                    safe_restart "$container" "CPU threshold exceeded"
                    set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
                    ((T1_RESTARTS++))
                    queue_notify "$container CPU ${CPU_NORM}% on $(hostname) — restarted" "warning"
                fi
            else
                set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
            fi
        done
    fi

    # HTTP responsiveness
    if [[ ${#WATCHDOG_CONTAINER_URLS[@]} -gt 0 ]]; then
        for container in "${!WATCHDOG_CONTAINER_URLS[@]}"; do
            URL="${WATCHDOG_CONTAINER_URLS[$container]}"
            if curl -sf --max-time "$CURL_TIMEOUT" "$URL" >/dev/null 2>&1; then
                set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
            else
                HTTP_STRIKES=$(get_strikes "${container}_http" "$WATCHDOG_STATE_FILE")
                HTTP_STRIKES=$(( HTTP_STRIKES + 1 ))
                set_strikes "${container}_http" "$HTTP_STRIKES" "$WATCHDOG_STATE_FILE"
                warn "$container — not responding at $URL (strike $HTTP_STRIKES/$RESP_FAIL_LIMIT)"
                ((T1_WARNINGS++))
                if [[ "$HTTP_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then
                    safe_restart "$container" "HTTP unresponsive"
                    set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
                    ((T1_RESTARTS++))
                    queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning"
                fi
            fi
        done
    fi

    # ── TIER 2 — Global Health Scan ─────────────────────────────────────────────────────────
    if [[ "$WATCHDOG_SCAN_ALL" == "true" ]]; then
        ALL_CONTAINERS=$(docker ps --format "{{.Names}}" 2>/dev/null)

        # Unhealthy containers
        if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then
            UNHEALTHY=$(docker ps --filter health=unhealthy --format "{{.Names}}" 2>/dev/null)
            while IFS= read -r container; do
                [[ -z "$container" ]] && continue
                [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
                is_skipped "$container" && continue
                error "$container — unhealthy"
                ((T2_WARNINGS++))
                result=0
                safe_restart "$container" "unhealthy health status" || result=$?
                [[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \
                    queue_notify "$container unhealthy on $(hostname) — restarted" "warning"
            done <<< "$UNHEALTHY"
        fi

        # OOM killed
        if [[ "$WATCHDOG_NOTIFY_OOM" == "true" ]]; then
            while IFS= read -r container; do
                [[ -z "$container" ]] && continue
                [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
                is_skipped "$container" && continue
                OOM=$(docker inspect -f '{{.State.OOMKilled}}' "$container" 2>/dev/null)
                if [[ "$OOM" == "true" ]]; then
                    error "$container — OOM killed"
                    ((T2_WARNINGS++))
                    result=0
                    safe_restart "$container" "OOM killed" || result=$?
                    [[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \
                        queue_notify "$container OOM killed on $(hostname) — restarted" "warning"
                fi
            done <<< "$ALL_CONTAINERS"
        fi

        # Crash loop detection
        if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then
            while IFS= read -r container; do
                [[ -z "$container" ]] && continue
                [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
                is_skipped "$container" && continue
                RESTART_COUNT=$(docker inspect -f '{{.RestartCount}}' "$container" 2>/dev/null || echo 0)
                PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d: -f2 || echo 0)
                set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE"
                if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then
                    ((T2_WARNINGS++))
                    if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then
                        error "$container — crash loop CRITICAL: $RESTART_COUNT restarts"
                        queue_notify "$container crash loop CRITICAL on $(hostname) — manual intervention needed" "critical"
                    else
                        warn "$container — restarted since last check (total: $RESTART_COUNT)"
                        queue_notify "$container restarted on $(hostname) — count: $RESTART_COUNT" "warning"
                    fi
                fi
            done <<< "$ALL_CONTAINERS"
        fi

        # Dead containers
        if [[ "$WATCHDOG_RESTART_DEAD" == "true" ]]; then
            DEAD=$(docker ps -a --filter status=dead --format "{{.Names}}" 2>/dev/null)
            while IFS= read -r container; do
                [[ -z "$container" ]] && continue
                [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
                is_skipped "$container" && continue
                error "$container — dead"
                ((T2_WARNINGS++))
                RESTART_COUNT=$(get_restart_count "$container")
                if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then
                    add_to_skip_list "$container" "dead — restarted $RESTART_COUNT times"
                elif [[ "$DRY_RUN" == false ]]; then
                    docker rm "$container" >/dev/null 2>&1
                    if docker start "$container" >/dev/null 2>&1; then
                        success "$container removed from dead state and restarted"
                        log_restart "$container"
                        ((T2_RESTARTS++))
                        queue_notify "$container was dead on $(hostname) — restarted" "warning"
                    fi
                fi
            done <<< "$DEAD"
        fi

        # Unexpected exits
        if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then
            CRASHED=$(docker ps -a \
                --filter status=exited \
                --format "{{.Names}}|{{.Status}}" 2>/dev/null | \
                grep -v "Exited (0)")
            while IFS='|' read -r container status; do
                [[ -z "$container" ]] && continue
                [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
                is_skipped "$container" && continue
                SKIP=false
                for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
                    [[ "$container" == "$req" ]] && SKIP=true && break
                done
                [[ "$SKIP" == true ]] && continue
                error "$container — $status (unexpected exit)"
                ((T2_WARNINGS++))
                result=0
                safe_restart "$container" "unexpected exit" || result=$?
                [[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \
                    queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning"
            done <<< "$CRASHED"
        fi
    fi

    # Send notifications if any events this cycle
    flush_notify

    # Only log summary if something happened — quiet when all healthy
    TOTAL_RESTARTS=$(( T1_RESTARTS + T2_RESTARTS ))
    TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS ))

    if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then
        CYCLE_END=$(date +%s)
        echo ""
        echo "━━━ $ICON_WATCHDOG Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
        echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings"
        echo "$ICON_WATCHDOG T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings"
        echo "$ICON_TIME      Duration: $(format_duration $(( CYCLE_END - CYCLE_START )))"
        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    else
        log "Cycle $CYCLE — all healthy ($(date '+%H:%M:%S'))"
        # Heartbeat — periodic proof of life
        if [[ "${DOCKER_WATCHDOG_HEARTBEAT:-true}" == true ]]; then
            HB_SECONDS=$(( ${DOCKER_WATCHDOG_HEARTBEAT_HOURS:-1} * 3600 ))
            UPTIME_SECONDS=$(( CYCLE * DOCKER_WATCHDOG_INTERVAL ))
            if [[ "$HB_SECONDS" -gt 0 ]] && (( UPTIME_SECONDS % HB_SECONDS < DOCKER_WATCHDOG_INTERVAL )) && [[ "$UPTIME_SECONDS" -gt 0 ]]; then
                HB_UPTIME_HR=$(( UPTIME_SECONDS / 3600 ))
                info "♥ docker_watchdog alive — ~${HB_UPTIME_HR}hr uptime ($(date '+%H:%M:%S'))"
            fi
        fi
    fi

    # Sleep until next cycle — interruptible by SIGTERM
    sleep "$DOCKER_WATCHDOG_INTERVAL" &
    wait $!

done