#!/bin/bash
# ==============================================================================================
# ================================= Docker Watchdog ============================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Two-tier self-healing container monitoring system. Called by
# watchdog_orchestrator.sh via cron every 15 minutes as a single-pass run.
# Tier 1 applies specific thresholds to explicitly configured containers.
# Tier 2 scans everything else for generic health problems. Silent on clean
# cycles, loud when something needs attention.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Tier 1 — Strict Per-Container Monitoring
#   Applies only to containers explicitly configured in host*.conf.
#
#   Memory hard limits    — immediate restart if container exceeds MB ceiling
#   Memory soft threshold — warn at SOFT_MEM_THRESHOLD % of limit (no restart)
#   CPU thresholds        — strike system: warn at SOFT_CPU_THRESHOLD, restart after
#                           CPU_FAIL_LIMIT consecutive strikes at HARD_CPU_THRESHOLD
#   HTTP responsiveness   — strike system: restart after RESP_FAIL_LIMIT consecutive
#                           failures against the configured endpoint
#   Required containers   — must always be running; strike system before restart;
#                           skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in window
#
# Tier 2 — Global Health Scan
#   Scans ALL running containers when WATCHDOG_SCAN_ALL=true.
#   Containers in WATCHDOG_SCAN_IGNORE are excluded.
#
#   Unhealthy status      — Docker HEALTHCHECK unhealthy → restart
#   OOM killed            — kernel OOM kill detected → restart + notify
#   Crash loop            — RestartCount climbing → notify; above WATCHDOG_CRASH_LIMIT
#                           → restart → skip list if restart limit hit
#   Dead containers       — remove + start (dead state cannot be restarted directly)
#   Unexpected exits      — non-zero exit code → restart
#
# Cross-Cutting Intelligence
#   Applies to both tiers on every cycle.
#
#   Startup grace period  — no restarts for WATCHDOG_STARTUP_GRACE seconds after boot
#   Dependency ordering   — dependency restarted first, dependent skipped this cycle
#   Restart loop protect  — skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in window
#   Skip list auto-clear  — removed when container seen running again
#   Notification batching — one summary per cycle, not one ping per event
#   Parity awareness      — skips restart actions during parity check
#   Timeout protection    — all docker commands wrapped in timeout
#   Docker daemon check   — each cycle begins with daemon health check; hung daemon →
#                           restart via rc.docker → stability_watchdog.sh escalates if needed
#   RAM emergency defer   — reads RW_STATE_FILE; stands down while
#                           resource_watchdog.sh is managing a RAM emergency
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Tiered Monitoring
#   Not all containers need the same monitoring strategy. Tier 1 gives explicit
#   control over the containers that matter most. Tier 2 is the catch-all that
#   requires no configuration and protects everything else.
#
# Strike vs Immediate
#   CPU spikes and HTTP failures are transient — brief spikes are normal during
#   transcoding or library scans. Memory leaks are not transient. CPU and HTTP
#   use a strike system to distinguish sustained problems from momentary ones.
#   Memory triggers immediate restart because a container at its ceiling is
#   actively leaking, not spiking.
#
# Loop Protection Over Persistence
#   A watchdog that keeps restarting a broken container is not helpful — it risks
#   making a database corruption worse. After WATCHDOG_CONTAINER_RESTART_LIMIT
#   attempts the container is skip-listed and the operator is notified. Automated
#   recovery stops. Human investigation begins.
#
# Dependency-Safe Ordering
#   When a container and its dependency are both down, restart the dependency
#   first and skip the dependent this cycle. Prevents false-alarm skip-listing
#   of containers whose only failure was starting before their dependency was ready.
#
# Silent When Healthy
#   Runs 96 times per day. Producing output on every clean cycle would make
#   logs useless. Output only when something needs attention — alive heartbeat
#   is handled by watchdog_orchestrator.sh, the caller, not this script.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root Enforcement
#   Container restarts and daemon service control require root.
#
# Docker Enabled Check
#   Exits cleanly when Docker is disabled in the platform's own settings. A
#   deliberately disabled Docker service is not a fault and must not be
#   "healed" by restarting the daemon.
#
# Docker Presence Check
#   Verifies the docker binary exists before the cycle begins.
#
# Lock Acquisition
#   Prevents concurrent execution via acquire_lock(). Safe at array start —
#   only one watchdog instance runs at a time.
#
# Host Detection
#   detect_hosts() identifies which server is running the script and aliases
#   all HOST*_WATCHDOG_* arrays to the correct host's values.
#
# Startup Grace Period
#   Restart actions suppressed for WATCHDOG_STARTUP_GRACE seconds after the
#   watchdog starts. Checks still run and log — only restarts are suppressed.
#   Prevents false-positive restarts while containers are still initialising.
#
# Restart Loop Protection
#   WATCHDOG_CONTAINER_RESTART_LIMIT restarts within WATCHDOG_CONTAINER_RESTART_WINDOW
#   hours triggers skip-listing and a critical notification. Skip list persists on
#   /boot/config/ — survives reboots intentionally. Auto-clears when container
#   is seen running again.
#
# Docker Daemon Health Check
#   First operation every cycle. Daemon not responding within DOCKER_TIMEOUT →
#   restart via platform_restart_service docker → verify recovery. If still hung:
#   log critical, skip cycle, and set daemon_confirmed_down so
#   stability_watchdog.sh owns any further escalation. The restart itself is
#   bounded by a 180 second timeout — a daemon stop can block for 30+ minutes on
#   a busy host, and the watchdog must not be held hostage to it.
#
# RAM Emergency Deferral
#   Reads RW_STATE_FILE each cycle. If resource_watchdog.sh has set
#   mem_shutdown_active=true, all restart logic defers until the flag clears.
#   Stale state guard: if file is >2 hours old with flag still set,
#   resource_watchdog.sh has likely stopped — watchdog resumes normal operation.
#
# Timeout Protection
#   All docker commands wrapped in timeout. Daemon hangs cannot stall the
#   watchdog and leave containers unmonitored between cycles.
#
# Stale Entry Pruning
#   Every cycle, entries naming a container that no longer exists are removed from
#   DOCKER_WATCHDOG_FAILED_FILE, DOCKER_WATCHDOG_INTENTIONAL_FILE, and the per-container
#   strike counts in WATCHDOG_STATE_FILE. None of these can clear themselves: every
#   clearing path requires seeing the container running again, which never happens once
#   it is uninstalled. Without pruning, one removed container makes this host report
#   unhealthy permanently. Reserved daemon_* keys in WATCHDOG_STATE_FILE are never pruned.
#   Pruning runs under --dry-run as well: the documented contract for that flag is that
#   nothing gets restarted, and dropping a record of a container that no longer exists is
#   housekeeping, not an action. It is idempotent — repeated runs converge.
#
# Notification Batching
#   Events collected across a full cycle and sent as a single summary.
#   Prevents notification floods when a shared dependency failure cascades.
#
# ==============================================================================================
# STATE FILES
# ==============================================================================================
#
#   WATCHDOG_STATE_FILE               — strike counts, daemon flags (STATE_DIR — survives reboots)
#   DOCKER_WATCHDOG_FAILED_FILE       — container skip list (STATE_DIR — survives reboots)
#   DOCKER_WATCHDOG_INTENTIONAL_FILE  — intentional stops list (STATE_DIR — survives reboots)
#   WATCHDOG_CONTAINER_RESTART_LOG    — restart history for loop detection (DATA_DIR)
#   RW_STATE_FILE                     — read-only: resource_watchdog RAM emergency flag
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# host*.conf
#
#   HOST*_WATCHDOG_CONTAINERS
#       Memory hard limits per container. Format: "ContainerName:LimitInMB"
#       Aliased by detect_hosts() → WATCHDOG_CONTAINERS
#
#   HOST*_WATCHDOG_CONTAINER_URLS
#       HTTP health check endpoints. Format: "ContainerName:http://host:port"
#       Aliased by detect_hosts() → WATCHDOG_CONTAINER_URLS
#
#   HOST*_WATCHDOG_CONTAINER_API_CHECKS
#       API liveness checks — deeper than HTTP. Format: "ContainerName:URL|APIKey"
#       Use endpoints that require a live DB round-trip (e.g. Emby /System/Info).
#       Aliased by detect_hosts() → WATCHDOG_CONTAINER_API_CHECKS
#
#   HOST*_WATCHDOG_REQUIRED_CONTAINERS
#       Containers that must always be running. Aliased by detect_hosts() →
#       WATCHDOG_REQUIRED_CONTAINERS
#
#   HOST*_WATCHDOG_SCAN_IGNORE
#       Containers excluded from Tier 2 global scan. Aliased by detect_hosts() →
#       WATCHDOG_SCAN_IGNORE
#
#   HOST*_WATCHDOG_DEPENDENCIES
#       Dependency ordering. Format: "Dependent:dep1 dep2". Aliased by
#       detect_hosts() → WATCHDOG_DEPENDENCIES
#
# master.conf
#
#   WATCHDOG_STARTUP_GRACE
#       Seconds before restart actions begin after watchdog starts (default: 600)
#
#   SOFT_MEM_THRESHOLD
#       Warn at this % of hard memory limit — no restart (default: 80)
#
#   SOFT_CPU_THRESHOLD / HARD_CPU_THRESHOLD / CPU_FAIL_LIMIT
#       CPU monitoring thresholds and strike limit
#
#   CURL_TIMEOUT / RESP_FAIL_LIMIT
#       HTTP health check timeout and consecutive failure limit
#
#   WATCHDOG_SCAN_ALL
#       Enable Tier 2 global health scan (default: true)
#
#   WATCHDOG_RESTART_UNHEALTHY / WATCHDOG_RESTART_DEAD / WATCHDOG_RESTART_CRASHED
#       Tier 2 action toggles
#
#   WATCHDOG_NOTIFY_OOM / WATCHDOG_NOTIFY_CRASHLOOP / WATCHDOG_CRASH_LIMIT
#       OOM and crash loop detection toggles and threshold
#
#   WATCHDOG_CONTAINER_RESTART_LIMIT / WATCHDOG_CONTAINER_RESTART_WINDOW
#       Restart loop protection: attempt limit and rolling window in hours
#
#   WATCHDOG_REQUIRED_STRIKE_LIMIT
#       Consecutive down-checks on a required container before a restart is attempted (default: 2)
#
#   WATCHDOG_BATCH_NOTIFY
#       Collect cycle events and send as one notification (default: true)
#
#   WATCHDOG_DAEMON_TIMEOUT / WATCHDOG_DAEMON_STRIKE_LIMIT / WATCHDOG_DAEMON_RESTART_WAIT
#       Docker daemon health check: command timeout, strikes before restart attempt,
#       seconds to wait after restart before verifying
#
#   DOCKER_WATCHDOG_INTENTIONAL_FILE
#       Path to intentional stops state file (STATE_DIR). Containers in this file
#       are never restarted by the watchdog, regardless of exit code.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# docker_watchdog.sh
#     Single-pass monitoring cycle — called by watchdog_orchestrator.sh every 15 min
#
# docker_watchdog.sh --dry-run
#     Run a full watchdog cycle without restarting anything. Shows what would
#     happen based on current container states. Use to verify configuration.
#
# docker_watchdog.sh --status
#     Show strike counts, skip list contents, intentional stops, grace period status,
#     RAM emergency deferral state, and last cycle timing. Then exit.
#
# docker_watchdog.sh --log
#     Verbose output — full detail for every container checked and every decision.
#     Use to debug why a container is or is not being restarted.
#
# docker_watchdog.sh --pause ContainerName
#     Add ContainerName to the intentional stops list. Watchdog will not strike or
#     restart it until it is seen running again or --resume is called. Persists across
#     reboots. Use when stopping a required container for planned maintenance.
#
# docker_watchdog.sh --resume ContainerName
#     Remove ContainerName from the intentional stops list. Normal watchdog monitoring
#     resumes on the next cycle. The container is not started — it remains stopped
#     until started manually.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

# Extract watchdog-specific flags not handled by common parse_args
WATCHDOG_PAUSE_CONTAINER=""
WATCHDOG_RESUME_CONTAINER=""
for (( _wdi=0; _wdi<${#PARSED_ARGS[@]}; _wdi++ )); do
    case "${PARSED_ARGS[$_wdi]}" in
        --pause)  ((_wdi++)); WATCHDOG_PAUSE_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;;
        --resume) ((_wdi++)); WATCHDOG_RESUME_CONTAINER="${PARSED_ARGS[$_wdi]:-}" ;;
    esac
done
unset _wdi

# ==============================================================================================
# ━━━ Setup — runs once at start ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

# Skip if another instance is running — no pile-up during long operations
acquire_lock

# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays
detect_hosts

if ! is_docker_enabled; then
    echo "Docker not enabled in Unraid settings — skipping cycle"
    exit 0
fi

if ! command -v docker >/dev/null 2>&1; then
    error "Docker not found — cannot start watchdog"
    exit 1
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"

log "$ICON_GEAR Config: grace=${WATCHDOG_STARTUP_GRACE}s mem-soft=${SOFT_MEM_THRESHOLD}% cpu-soft=${SOFT_CPU_THRESHOLD}% cpu-hard=${HARD_CPU_THRESHOLD}% cpu-limit=${CPU_FAIL_LIMIT} http-limit=${RESP_FAIL_LIMIT} daemon-timeout=${DOCKER_TIMEOUT}s"
log "$ICON_CONTAINERS Tier1:  watched=${#WATCHDOG_CONTAINERS[@]} required=${#WATCHDOG_REQUIRED_CONTAINERS[@]} urls=${#WATCHDOG_CONTAINER_URLS[@]} restart-limit=${WATCHDOG_CONTAINER_RESTART_LIMIT}/${WATCHDOG_CONTAINER_RESTART_WINDOW}h"

# Ensure state files exist
touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \
      "$DOCKER_WATCHDOG_FAILED_FILE" "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null

# Timeout for all docker commands — configurable via WATCHDOG_DAEMON_TIMEOUT in master.conf
DOCKER_TIMEOUT="${WATCHDOG_DAEMON_TIMEOUT:-20}"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST       Identity:       $MY_ID ($LOCAL_SERVER_NAME)"
    _watched="${!WATCHDOG_CONTAINERS[*]}"; echo "$ICON_CONTAINERS Watched:        ${_watched:-none}"
    echo "$ICON_CONTAINERS Required:       ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}"
    _intentional=$(cat "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
    echo "$ICON_SKIP       Intentional:    ${_intentional:-none}"
    echo "$ICON_WATCHDOG   Scan all:       $WATCHDOG_SCAN_ALL"
    echo "$ICON_WATCHDOG   Ignore:         ${WATCHDOG_SCAN_IGNORE[*]:-none}"
    echo "$ICON_WATCHDOG   Schedule:       every 15 min (cron via watchdog_orchestrator)"
    echo "$ICON_WATCHDOG   Startup grace:  ${WATCHDOG_STARTUP_GRACE}s"
    echo "$ICON_WATCHDOG   Restart limit:  $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
    echo "$ICON_WATCHDOG   Batch notify:   $WATCHDOG_BATCH_NOTIFY"
    echo "$ICON_WATCHDOG   Docker timeout: ${DOCKER_TIMEOUT}s"
    echo "$ICON_GEAR       Dry Run:        $DRY_RUN"
    UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
    echo "$ICON_TIME        System uptime:  $(format_duration $UPTIME_S)"
    [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]] && \
        warn "Within startup grace period — restarts suppressed"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ── Intentional stop management — --pause / --resume ─────────────────────────────────────────
if [[ -n "$WATCHDOG_PAUSE_CONTAINER" || -n "$WATCHDOG_RESUME_CONTAINER" ]]; then
    if [[ -n "$WATCHDOG_PAUSE_CONTAINER" ]]; then
        if grep -q "^${WATCHDOG_PAUSE_CONTAINER}$" "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null; then
            warn "$WATCHDOG_PAUSE_CONTAINER already in intentional stops"
        else
            echo "$WATCHDOG_PAUSE_CONTAINER" >> "$DOCKER_WATCHDOG_INTENTIONAL_FILE"
            success "$WATCHDOG_PAUSE_CONTAINER added to intentional stops — watchdog will not restart it"
        fi
    fi
    if [[ -n "$WATCHDOG_RESUME_CONTAINER" ]]; then
        sed -i "/^${WATCHDOG_RESUME_CONTAINER}$/d" "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null
        success "$WATCHDOG_RESUME_CONTAINER removed from intentional stops — normal monitoring resumes next cycle"
    fi
    exit 0
fi

# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# Get strike count for a container from state file — wraps common.sh's wd_state_get()
get_strikes() {
    wd_state_get "$1" "$2"
}

# Set strike count for a container in state file — wraps common.sh's wd_state_set()
set_strikes() {
    local container="$1" count="$2" file="$3"
    wd_state_set "$container" "$count" "$file"
}

# Check if container is on the persistent skip list
is_skipped() {
    grep -q "^${1}$" "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null
}

# Add container to persistent skip list — manual intervention required to recover
add_to_skip_list() {
    local container="$1" reason="$2"
    if ! is_skipped "$container"; then
        echo "$container" >> "$DOCKER_WATCHDOG_FAILED_FILE"
        error "$container added to skip list — $reason"
        queue_notify "$container added to skip list on $(hostname) — $reason — manual intervention needed" "critical"
    fi
}

# Remove container from skip list — called when container is seen running again
remove_from_skip_list() {
    sed -i "/^${1}$/d" "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null
    warn "$1 recovered — removed from skip list ✅"
}

# Check if container is in the intentional stops list
is_intentional_stop() {
    grep -q "^${1}$" "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null
}

# Remove from intentional stops — called when container is seen running again
clear_intentional_stop() {
    if grep -q "^${1}$" "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null; then
        sed -i "/^${1}$/d" "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null
        warn "$1 running — removed from intentional stops ✅"
    fi
}

# Log a restart event to the rolling restart history file
log_restart() {
    local container="$1"
    local now
    now=$(date '+%Y-%m-%d %H:%M:%S')
    local cutoff
    cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S')
    echo "${container}|${now}" >> "$WATCHDOG_CONTAINER_RESTART_LOG"
    # Trim entries older than the rolling window
    local tmp="${WATCHDOG_CONTAINER_RESTART_LOG}.tmp"
    awk -F'|' -v cutoff="$cutoff" '$2 >= cutoff' \
        "$WATCHDOG_CONTAINER_RESTART_LOG" > "$tmp" && \
        mv "$tmp" "$WATCHDOG_CONTAINER_RESTART_LOG"
}

# Get number of times a container was restarted within the rolling window
get_restart_count() {
    local container="$1"
    local cutoff
    cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S')
    awk -F'|' -v c="$container" -v cutoff="$cutoff" \
        '$1==c && $2>=cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l
}

# Check if all dependencies of a container are currently running.
# Returns 0 if all deps running (or no deps), 1 if any dep is down.
dependencies_satisfied() {
    local container="$1"
    local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
    [[ -z "$deps" ]] && return 0
    for dep in $deps; do
        local status
        status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null)
        if [[ "$status" != "true" ]]; then
            warn "$container — dependency $dep is not running — skipping restart this cycle"
            return 1
        fi
    done
    return 0
}

# Safe restart with all guards:
#   - Restart loop protection (skip list after limit)
#   - Dependency check (don't restart if deps down)
#   - Startup grace period (no restarts while booting)
#   - Dry run support
#   - Timeout protection on docker restart
#
# Returns: 0=restarted, 1=skipped, 2=added to skip list
safe_restart() {
    local container="$1" reason="$2"

    # Restart loop protection — skip list if over limit
    local restart_count
    restart_count=$(get_restart_count "$container")
    if [[ "$restart_count" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then
        add_to_skip_list "$container" \
            "restarted $restart_count times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
        return 2
    fi

    # Dependency check
    dependencies_satisfied "$container" || return 1

    # Startup grace period
    local uptime_s
    uptime_s=$(awk '{print int($1)}' /proc/uptime)
    if [[ "$uptime_s" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
        warn "$container — within startup grace period (${uptime_s}s < ${WATCHDOG_STARTUP_GRACE}s) — skipping"
        return 1
    fi

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would restart $container ($reason)"
        return 0
    fi

    log "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]"
    if timeout "$DOCKER_TIMEOUT" docker restart "$container" >/dev/null 2>&1; then
        success "$ICON_STARTED $container restarted"
        log_restart "$container"
        return 0
    else
        error "Failed to restart $container (timeout or error)"
        return 1
    fi
}

# Queue a notification event for batch sending at end of cycle
queue_notify() {
    local message="$1" severity="${2:-warning}"
    NOTIFY_EVENTS+=("${severity}|${message}")
    log "Queued: $message"
}

# Send all queued notifications — one batched summary or individual per event
flush_notify() {
    [[ ${#NOTIFY_EVENTS[@]} -eq 0 ]] && return
    if [[ "$WATCHDOG_BATCH_NOTIFY" == "true" ]]; then
        local highest_severity="normal"
        local messages=()
        for event in "${NOTIFY_EVENTS[@]}"; do
            local sev="${event%%|*}" msg="${event#*|}"
            messages+=("$msg")
            [[ "$sev" == "critical" ]] && highest_severity="warning"
            [[ "$sev" == "warning" && "$highest_severity" == "normal" ]] && highest_severity="warning"
        done
        local summary
        summary=$(printf '%s. ' "${messages[@]}")
        notify "Docker Watchdog on $(hostname) — ${#NOTIFY_EVENTS[@]} event(s): $summary" \
            "Docker Watchdog" "$highest_severity"
    else
        for event in "${NOTIFY_EVENTS[@]}"; do
            local sev="${event%%|*}" msg="${event#*|}"
            [[ "$sev" == "critical" ]] && sev="warning"
            notify "$msg" "Docker Watchdog" "$sev"
        done
    fi
    NOTIFY_EVENTS=()
}

# is_parity_running — delegates to adapter
is_parity_running() { platform_is_maintenance_running; }

# ==============================================================================================
# ── DOCKER DAEMON HEALTH CHECK ────────────────────────────────────────────────────────────────
# ==============================================================================================
# Checks Docker daemon responsiveness at the start of every cycle.
# A hung daemon makes all container operations useless — check first, short-circuit if down.
#
# Strike system:
#   Each consecutive failed check adds a strike
#   At WATCHDOG_DAEMON_STRIKE_LIMIT → attempt daemon restart via rc.docker
#   After restart → wait WATCHDOG_DAEMON_RESTART_WAIT seconds → verify
#   If verified → clear strikes, continue cycle ✅
#   If still hung → notify critical, skip cycle → stability_watchdog.sh escalates from here
#
# Returns: 0 = daemon healthy | 1 = daemon down, skip this cycle

WATCHDOG_DAEMON_STRIKE_LIMIT="${WATCHDOG_DAEMON_STRIKE_LIMIT:-3}"    # consecutive failed checks before restart attempt (master.conf)
WATCHDOG_DAEMON_RESTART_WAIT="${WATCHDOG_DAEMON_RESTART_WAIT:-900}"  # seconds to wait after restart before verifying (master.conf)
# Loaded from state file — persists across single-pass runs
WATCHDOG_DAEMON_STRIKES=$(get_strikes "daemon_strikes" "$WATCHDOG_STATE_FILE")
WATCHDOG_DAEMON_STRIKES="${WATCHDOG_DAEMON_STRIKES//[^0-9]/}"; WATCHDOG_DAEMON_STRIKES="${WATCHDOG_DAEMON_STRIKES:-0}"
_dr_raw=$(get_strikes "daemon_restarted_flag" "$WATCHDOG_STATE_FILE")
[[ "$_dr_raw" == "true" ]] && WATCHDOG_DAEMON_RESTARTED=true || WATCHDOG_DAEMON_RESTARTED=false

# ==============================================================================================
# ── SYSTEM WATCHDOG COORDINATION ──────────────────────────────────────────────────────────────
# ==============================================================================================
# Reads system_watchdog.sh state file to check if a RAM emergency shutdown is active.
# During RAM emergency: system_watchdog.sh has stopped non-essential containers to free RAM.
# docker_watchdog.sh must not restart them — that would undo the emergency shutdown and
# prevent RAM from recovering, creating an infinite restart/shutdown loop.
#
# Returns:
#   0 = normal — run all checks
#   1 = RAM emergency active — defer container management this cycle

check_resource_watchdog_state() {
    # Returns 0 = normal operation | 1 = defer, resource_watchdog RAM emergency active
    local state_file="$RW_STATE_FILE"

    # No state file = resource_watchdog not yet run — assume normal
    [[ ! -f "$state_file" ]] && return 0

    local mem_shutdown
    mem_shutdown=$(grep "^mem_shutdown_active=" "$state_file" 2>/dev/null | cut -d= -f2)

    [[ "$mem_shutdown" != "true" ]] && return 0

    # ── Stale state guard ─────────────────────────────────────────────────────────────────────
    # If mem_shutdown_active=true but state file hasn't been updated in > 2 hours,
    # resource_watchdog.sh may not be running — don't defer indefinitely on stale state.
    local state_mtime now age_seconds stale_limit=7200  # 2 hours
    state_mtime=$(stat -c %Y "$state_file" 2>/dev/null || echo 0)
    now=$(date +%s)
    age_seconds=$(( now - state_mtime ))

    if [[ "$age_seconds" -gt "$stale_limit" ]]; then
        warn "mem_shutdown_active=true but state file is ${age_seconds}s old — may be stale"
        warn "resource_watchdog.sh may not be running — resuming normal container management"
        return 0  # Resume normal — don't defer indefinitely on stale state
    fi

    return 1  # Defer — RAM emergency confirmed and state is fresh
}

check_docker_daemon() {
    # docker info is more definitive than docker ps for daemon health
    if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
        if [[ "$WATCHDOG_DAEMON_STRIKES" -gt 0 ]]; then
            log "$ICON_STARTED Docker daemon recovered — clearing strikes"
            queue_notify "Docker daemon recovered on $(hostname)" "normal"
            WATCHDOG_DAEMON_STRIKES=0
            WATCHDOG_DAEMON_RESTARTED=false
            set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE"
            set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE"
        fi
        # Always clear confirmed-down flag when daemon is healthy — unconditional so it
        # clears correctly after a reboot (strikes reset to 0 but flag persists on flash).
        set_strikes "daemon_confirmed_down" "false" "$WATCHDOG_STATE_FILE"
        return 0
    fi

    WATCHDOG_DAEMON_STRIKES=$(( WATCHDOG_DAEMON_STRIKES + 1 ))
    set_strikes "daemon_strikes" "$WATCHDOG_DAEMON_STRIKES" "$WATCHDOG_STATE_FILE"
    warn "$ICON_WATCHDOG Docker daemon not responding (strike $WATCHDOG_DAEMON_STRIKES/$WATCHDOG_DAEMON_STRIKE_LIMIT)"

    if [[ "$WATCHDOG_DAEMON_STRIKES" -lt "$WATCHDOG_DAEMON_STRIKE_LIMIT" ]]; then
        warn "Skipping monitoring cycle — waiting for daemon to recover"
        return 1
    fi

    if [[ "$WATCHDOG_DAEMON_RESTARTED" == true ]]; then
        # Restart was already attempted last cycle and daemon is still down.
        # Write confirmed-down flag — stability_watchdog reads this and strikes toward reboot.
        error "Docker daemon still unresponsive after restart attempt"
        set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE"
        queue_notify "Docker daemon hung on $(hostname) — restart failed — stability_watchdog escalating" "critical"
        flush_notify
        return 1
    fi

    error "Docker daemon unresponsive — attempting restart"
    notify "Docker daemon hung on $(hostname) — attempting restart" "Docker Watchdog" "warning"

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would restart Docker daemon via platform_restart_service docker"
        return 1
    fi

    # timeout 180: Docker daemon stop can block for 30+ min on a busy host.
    WATCHDOG_DAEMON_RESTARTED=true
    set_strikes "daemon_restarted_flag" "true" "$WATCHDOG_STATE_FILE"
    if timeout 180 platform_restart_service docker; then
        log "Docker daemon restart issued — waiting ${WATCHDOG_DAEMON_RESTART_WAIT}s..."
        sleep "$WATCHDOG_DAEMON_RESTART_WAIT"

        if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
            success "Docker daemon restarted successfully ✅"
            notify "Docker daemon restarted successfully on $(hostname)" "Docker Watchdog" "normal"
            WATCHDOG_DAEMON_STRIKES=0
            WATCHDOG_DAEMON_RESTARTED=false
            set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE"
            set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE"
            set_strikes "daemon_confirmed_down" "false" "$WATCHDOG_STATE_FILE"
            return 0
        else
            # Restart issued but daemon still down — flag for stability_watchdog on next cycle
            error "Docker daemon did not recover after restart"
            set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE"
            queue_notify "Docker daemon restart failed on $(hostname) — stability_watchdog escalating" "critical"
            flush_notify
            return 1
        fi
    else
        # platform_restart_service returned non-zero — flag immediately, stability_watchdog escalates
        error "Failed to issue Docker daemon restart — platform_restart_service docker failed"
        set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE"
        queue_notify "Docker daemon restart command failed on $(hostname) — stability_watchdog escalating" "critical"
        flush_notify
        return 1
    fi
}

# ==============================================================================================
# ━━━ Single-Pass Monitoring Run ━━━
# ==============================================================================================
echo "━━━ $ICON_WATCHDOG Docker Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

CYCLE_START=$(date +%s)

    # ── Per-run state ─────────────────────────────────────────────────────────────────────────
    NOTIFY_EVENTS=()
    T1_RESTARTS=0
    T1_WARNINGS=0
    T2_RESTARTS=0
    T2_WARNINGS=0

    # OOM handled set — tracks containers already handled for OOM this cycle
    # Prevents restart loop from OOMKilled flag persisting after restart
    declare -A OOM_HANDLED

    # Rebuild ignore map each cycle (config may have changed)
    declare -A IGNORE_MAP
    for c in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do
        [[ -n "$c" ]] && IGNORE_MAP["$c"]=1
    done

    # ── Skip list and intentional stops visibility ───────────────────────────────────────────
    # Prune entries for containers that no longer exist at all (uninstalled/removed) from both
    # state files. Neither remove_from_skip_list() nor clear_intentional_stop() can ever fire
    # for one of these — both only trigger when a container is "seen running again," which never
    # happens for something that's been uninstalled — so without this an entry nags every cycle
    # forever (confirmed live 2026-07-19: Healarr, uninstalled weeks earlier, still flagged every
    # run). Snapshot into an array first — sed -i rewriting the same file a `while read < file`
    # loop is still iterating is the classic gotcha this avoids.
    for _prune_file in "$DOCKER_WATCHDOG_FAILED_FILE" "$DOCKER_WATCHDOG_INTENTIONAL_FILE"; do
        [[ -s "$_prune_file" ]] || continue
        mapfile -t _prune_snapshot < "$_prune_file"
        for _prune_container in "${_prune_snapshot[@]}"; do
            [[ -z "$_prune_container" ]] && continue
            if ! timeout "$DOCKER_TIMEOUT" docker inspect "$_prune_container" &>/dev/null; then
                sed -i "/^${_prune_container}$/d" "$_prune_file" 2>/dev/null
                warn "$_prune_container no longer exists — removed from $(basename "$_prune_file")"
            fi
        done
    done
    unset _prune_file _prune_snapshot _prune_container

    # Same failure, different file. WATCHDOG_STATE_FILE cannot join the loop above: it stores
    # key:value rather than one name per line, and it holds reserved daemon_* keys alongside the
    # per-container strike counts. A strike entry for an uninstalled container never clears on its
    # own — set_strikes "$container" 0 only fires when the container is seen running again, which
    # by definition never happens — so the count sits there and every consumer that treats a
    # non-zero strike as unhealthy reports this host unhealthy forever. Confirmed live 2026-08-02:
    # claudeclaw_docker at 2119 strikes long after removal, which is what kept the monitor page's
    # watchdog summary from ever going green.
    if [[ -s "$WATCHDOG_STATE_FILE" ]]; then
        mapfile -t _prune_snapshot < "$WATCHDOG_STATE_FILE"
        for _prune_line in "${_prune_snapshot[@]}"; do
            _prune_container="${_prune_line%%:*}"
            [[ -z "$_prune_container" ]] && continue
            [[ "$_prune_container" == daemon_* ]] && continue   # reserved keys, not containers
            if ! timeout "$DOCKER_TIMEOUT" docker inspect "$_prune_container" &>/dev/null; then
                sed -i "/^${_prune_container}:/d" "$WATCHDOG_STATE_FILE" 2>/dev/null
                warn "$_prune_container no longer exists — removed from $(basename "$WATCHDOG_STATE_FILE")"
            fi
        done
        unset _prune_snapshot _prune_line _prune_container
    fi

    _skip_contents=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
    [[ -n "$_skip_contents" ]] && warn "$ICON_SKIP Skip list active: $_skip_contents — manual intervention needed"
    _intentional_contents=$(cat "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
    [[ -n "$_intentional_contents" ]] && warn "$ICON_SKIP Intentional stops: $_intentional_contents — watchdog will not restart these"

    # ── Docker daemon health check — first check every run ──────────────────────────────────
    # If daemon is hung all container operations will fail — check first, skip run if down
    if ! check_docker_daemon; then
        exit 0
    fi

    # ── Parity check — skip restarts during parity ───────────────────────────────────────────
    if is_parity_running; then
        echo "Parity in progress — restart actions skipped ($(date '+%H:%M:%S'))"
        exit 0
    fi

    # ── RAM emergency check — resource_watchdog.sh managing containers ────────────────────────
    # If resource_watchdog.sh has triggered a hard RAM shutdown, defer all container
    # management this run to prevent undoing the emergency stop and re-pressuring RAM.
    if ! check_resource_watchdog_state; then
        MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
        MEM_GB=$(( MEM_KB / 1024 / 1024 ))
        warn "RAM emergency active (${MEM_GB}GB free) — resource_watchdog.sh managing containers"
        warn "Deferring all container restart logic this run"
        log "Waiting for RAM to recover above ${RW_RAM_RECOVER_GB:-20}GB before resuming"
        exit 0
    fi

    # ── Startup grace period ──────────────────────────────────────────────────────────────────
    UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
    IN_GRACE_PERIOD=false
    [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]] && IN_GRACE_PERIOD=true

    # ==========================================================================================
    # ── TIER 1 — Strict Monitoring ────────────────────────────────────────────────────────────
    # ==========================================================================================

    # ── Required containers ───────────────────────────────────────────────────────────────────
    # Must always be running — strike system before restart, skip list after limit
    if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
        for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
            [[ -z "$container" ]] && continue

            STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
                "$container" 2>/dev/null)

            if is_skipped "$container"; then
                if [[ "$STATUS" == "true" ]]; then
                    # Container recovered — remove from skip list and clear its history
                    remove_from_skip_list "$container"
                    set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
                    sed -i "/^${container}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
                    queue_notify "$container recovered on $(hostname) — removed from skip list" "normal"
                else
                    warn "$container — on skip list, manual intervention needed"
                fi
                continue
            fi

            if [[ "$STATUS" == "true" ]]; then
                # Running — clear strikes and any intentional-stop flag set from a prior cycle
                set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
                clear_intentional_stop "$container"
                log "$ICON_RUNNING $container — running ✅"
            else
                # Check intentional stops first — explicit operator instruction beats everything
                if is_intentional_stop "$container"; then
                    log "$container — intentionally stopped (on pause list) — skipping"
                    continue
                fi

                # Exit code 0 = cleanly stopped (docker stop, Unraid UI stop, clean shutdown).
                # Don't strike or restart — operator almost certainly stopped it on purpose.
                # Use --pause to make this permanent across reboots.
                LAST_EXIT=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
                    '{{.State.ExitCode}}' "$container" 2>/dev/null)
                if [[ "$LAST_EXIT" == "0" ]]; then
                    log "$container — stopped cleanly (exit 0) — treating as intentional; use --pause to suppress permanently"
                    set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
                    continue
                fi

                STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
                STRIKES=$(( STRIKES + 1 ))
                set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
                warn "$container — not running, exit ${LAST_EXIT} (strike $STRIKES/${WATCHDOG_REQUIRED_STRIKE_LIMIT:-2})"
                ((T1_WARNINGS++))

                if [[ "$STRIKES" -ge "${WATCHDOG_REQUIRED_STRIKE_LIMIT:-2}" ]]; then
                    result=0
                    safe_restart "$container" "required container down (exit ${LAST_EXIT})" || result=$?
                    case $result in
                        0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
                           ((T1_RESTARTS++))
                           queue_notify "$container was down (exit ${LAST_EXIT}) and restarted on $(hostname)" "warning" ;;
                        2) : ;; # Added to skip list — already notified
                        *) queue_notify "$container failed to restart on $(hostname)" "warning" ;;
                    esac
                fi
            fi
        done
    fi

    # ── Memory and CPU monitoring ─────────────────────────────────────────────────────────────
    if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
        STATS=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
            --format "{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}" 2>/dev/null)
        TOTAL_CORES=$(nproc 2>/dev/null || echo 1)

        for container in "${!WATCHDOG_CONTAINERS[@]}"; do
            MEM_LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
            CONTAINER_STATS=$(echo "$STATS" | grep "^${container}|" | head -1)
            [[ -z "$CONTAINER_STATS" ]] && continue

            # ── Memory ────────────────────────────────────────────────────────────────────────
            MEM_USAGE=$(echo "$CONTAINER_STATS" | cut -d'|' -f2 | awk '{print $1}')
            MEM_UNIT=$(echo "$MEM_USAGE" | grep -oE '[A-Za-z]+')
            MEM_VALUE=$(echo "$MEM_USAGE" | grep -oE '[0-9.]+')

            case "$MEM_UNIT" in
                GiB|GB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE * 1024}") ;;
                MiB|MB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE}") ;;
                KiB|KB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE / 1024}") ;;
                *)       MEM_MB=0 ;;
            esac

            # Soft memory threshold — warn when approaching hard limit
            SOFT_MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_LIMIT_MB * $SOFT_MEM_THRESHOLD / 100}")
            if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then
                # Hard limit exceeded — immediate restart
                error "$container — memory ${MEM_MB}MB exceeded hard limit ${MEM_LIMIT_MB}MB"
                safe_restart "$container" "memory hard limit exceeded"
                ((T1_RESTARTS++))
                queue_notify "$container exceeded memory hard limit on $(hostname) — restarted" "warning"
            elif [[ "$MEM_MB" -ge "$SOFT_MEM_MB" ]]; then
                # Soft threshold — warn only, no restart
                warn "$container — memory ${MEM_MB}MB approaching limit (${SOFT_MEM_THRESHOLD}% of ${MEM_LIMIT_MB}MB)"
                ((T1_WARNINGS++))
            fi

            # ── CPU ───────────────────────────────────────────────────────────────────────────
            CPU_RAW=$(echo "$CONTAINER_STATS" | cut -d'|' -f3 | tr -d '%')
            CPU_NORM=$(awk "BEGIN {printf \"%.1f\", $CPU_RAW / $TOTAL_CORES}")
            CPU_INT=$(printf "%.0f" "$CPU_NORM")

            if [[ "$CPU_INT" -ge "$HARD_CPU_THRESHOLD" ]]; then
                # Hard CPU threshold — strike system → restart
                CPU_STRIKES=$(get_strikes "${container}_cpu" "$WATCHDOG_STATE_FILE")
                CPU_STRIKES=$(( CPU_STRIKES + 1 ))
                set_strikes "${container}_cpu" "$CPU_STRIKES" "$WATCHDOG_STATE_FILE"
                warn "$container — CPU ${CPU_NORM}% (strike $CPU_STRIKES/$CPU_FAIL_LIMIT)"
                if [[ "$CPU_STRIKES" -ge "$CPU_FAIL_LIMIT" ]]; then
                    safe_restart "$container" "CPU hard threshold exceeded ${CPU_NORM}%"
                    set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
                    ((T1_RESTARTS++))
                    queue_notify "$container CPU ${CPU_NORM}% on $(hostname) — restarted" "warning"
                fi
            elif [[ "$CPU_INT" -ge "$SOFT_CPU_THRESHOLD" ]]; then
                # Soft CPU threshold — warn only, no restart, clear strikes
                warn "$container — CPU ${CPU_NORM}% (above soft threshold ${SOFT_CPU_THRESHOLD}%)"
                ((T1_WARNINGS++))
                set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
            else
                # Normal — clear CPU strikes
                set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
                log "$container — CPU ${CPU_NORM}% | MEM ${MEM_MB}MB / ${MEM_LIMIT_MB}MB ✅"
            fi
        done
    fi

    # ── HTTP responsiveness ───────────────────────────────────────────────────────────────────
    if [[ ${#WATCHDOG_CONTAINER_URLS[@]} -gt 0 ]]; then
        for container in "${!WATCHDOG_CONTAINER_URLS[@]}"; do
            URL="${WATCHDOG_CONTAINER_URLS[$container]}"
            if curl -sf --max-time "$CURL_TIMEOUT" "$URL" >/dev/null 2>&1; then
                set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
            else
                HTTP_STRIKES=$(get_strikes "${container}_http" "$WATCHDOG_STATE_FILE")
                HTTP_STRIKES=$(( HTTP_STRIKES + 1 ))
                set_strikes "${container}_http" "$HTTP_STRIKES" "$WATCHDOG_STATE_FILE"
                warn "$container — not responding at $URL (strike $HTTP_STRIKES/$RESP_FAIL_LIMIT)"
                ((T1_WARNINGS++))
                if [[ "$HTTP_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then
                    result=0
                    safe_restart "$container" "HTTP unresponsive at $URL" || result=$?
                    if [[ $result -eq 0 ]]; then
                        set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
                        ((T1_RESTARTS++))
                        queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning"
                    fi
                fi
            fi
        done
    fi

    # ── API liveness checks ───────────────────────────────────────────────────────────────────
    # Catches containers that serve HTTP 200 but are internally frozen (DB lock, deadlocked
    # thread, etc.). Endpoint must require a live DB round-trip to respond successfully.
    # Format per entry: "URL|APIKey"
    if [[ ${#WATCHDOG_CONTAINER_API_CHECKS[@]} -gt 0 ]]; then
        for container in "${!WATCHDOG_CONTAINER_API_CHECKS[@]}"; do
            IFS='|' read -r _api_url _api_key <<< "${WATCHDOG_CONTAINER_API_CHECKS[$container]}"

            # Skip entirely if key is absent or a placeholder — check is optional protection
            if [[ -z "$_api_key" || "$_api_key" == "YOUR_API_KEY"* || "$_api_key" == "placeholder"* ]]; then
                log "$container — API check skipped (no key configured)"
                continue
            fi

            _api_http=$(curl -s --max-time "$CURL_TIMEOUT" \
                -H "X-Emby-Token: ${_api_key}" \
                -o /tmp/_varaverk_api_check \
                -w "%{http_code}" \
                "$_api_url" 2>/dev/null)
            _resp=$(cat /tmp/_varaverk_api_check 2>/dev/null)

            # 401/403 = wrong key — skip silently, don't penalise the container
            if [[ "$_api_http" == "401" || "$_api_http" == "403" ]]; then
                log "$container — API check skipped (HTTP $_api_http — key may be wrong or revoked)"
                continue
            fi

            # Object-shaped responses (e.g. /System/Info) pass via ServerName/Id/Version.
            # Array-shaped responses (e.g. /Users) pass on any valid array — indexing an
            # array with a string key would itself error in jq, so branch on type first.
            if echo "$_resp" | jq -e 'if type == "array" then true else (.ServerName // .Id // .Version) != null end' >/dev/null 2>&1; then
                set_strikes "${container}_api" 0 "$WATCHDOG_STATE_FILE"
            else
                API_STRIKES=$(get_strikes "${container}_api" "$WATCHDOG_STATE_FILE")
                API_STRIKES=$(( API_STRIKES + 1 ))
                set_strikes "${container}_api" "$API_STRIKES" "$WATCHDOG_STATE_FILE"
                warn "$container — API unresponsive at $_api_url (strike $API_STRIKES/$RESP_FAIL_LIMIT)"
                ((T1_WARNINGS++))
                if [[ "$API_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then
                    result=0
                    safe_restart "$container" "API unresponsive at $_api_url" || result=$?
                    if [[ $result -eq 0 ]]; then
                        set_strikes "${container}_api" 0 "$WATCHDOG_STATE_FILE"
                        ((T1_RESTARTS++))
                        queue_notify "$container API unresponsive at $_api_url on $(hostname) — restarted" "warning"
                    fi
                fi
            fi
        done
    fi

    # ==========================================================================================
    # ── TIER 2 — Global Health Scan ───────────────────────────────────────────────────────────
    # ==========================================================================================
    if [[ "$WATCHDOG_SCAN_ALL" == "true" ]]; then

        ALL_CONTAINERS=$(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null)

        # ── Unhealthy containers ──────────────────────────────────────────────────────────────
        if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then
            UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \
                --filter health=unhealthy --format "{{.Names}}" 2>/dev/null)
            while IFS= read -r container; do
                [[ -z "$container" ]] && continue
                [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
                is_skipped "$container" && continue
                error "$container — Docker HEALTHCHECK unhealthy"
                ((T2_WARNINGS++))
                result=0
                safe_restart "$container" "unhealthy health status" || result=$?
                if [[ $result -eq 0 ]]; then
                    ((T2_RESTARTS++))
                    queue_notify "$container unhealthy on $(hostname) — restarted" "warning"
                fi
            done <<< "$UNHEALTHY"
        fi

        # ── OOM killed ────────────────────────────────────────────────────────────────────────
        # OOMKilled flag persists after restart — track handled containers per-cycle
        # to prevent the same container triggering a restart loop every cycle
        if [[ "$WATCHDOG_NOTIFY_OOM" == "true" ]]; then
            while IFS= read -r container; do
                [[ -z "$container" ]] && continue
                [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
                is_skipped "$container" && continue
                [[ -n "${OOM_HANDLED[$container]:-}" ]] && continue  # already handled this cycle
                OOM=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
                    '{{.State.OOMKilled}}' "$container" 2>/dev/null)
                if [[ "$OOM" == "true" ]]; then
                    error "$container — OOM killed by kernel"
                    ((T2_WARNINGS++))
                    OOM_HANDLED["$container"]=1
                    result=0
                    safe_restart "$container" "OOM killed" || result=$?
                    if [[ $result -eq 0 ]]; then
                        ((T2_RESTARTS++))
                        queue_notify "$container OOM killed on $(hostname) — restarted" "warning"
                    fi
                fi
            done <<< "$ALL_CONTAINERS"
        fi

        # ── Crash loop detection ──────────────────────────────────────────────────────────────
        # Tracks Docker's own RestartCount climbing between cycles.
        # Below WATCHDOG_CRASH_LIMIT: notify only — Docker's restart policy is handling it.
        # At or above WATCHDOG_CRASH_LIMIT: safe_restart() which will add to skip list
        # if WATCHDOG_CONTAINER_RESTART_LIMIT is also hit — ensures eventual quarantine.
        if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then
            while IFS= read -r container; do
                [[ -z "$container" ]] && continue
                [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
                is_skipped "$container" && continue
                RESTART_COUNT=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
                    '{{.RestartCount}}' "$container" 2>/dev/null || echo 0)
                EXIT_CODE=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
                    '{{.State.ExitCode}}' "$container" 2>/dev/null || echo 0)
                PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" \
                    2>/dev/null | cut -d: -f2 || echo 0)
                set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE"
                # Skip clean-exit containers (exit 0 = completed, not crashed)
                [[ "$EXIT_CODE" == "0" ]] && continue
                if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then
                    ((T2_WARNINGS++))
                    if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then
                        error "$container — crash loop CRITICAL: $RESTART_COUNT restarts"
                        # Attempt restart via safe_restart — will add to skip list if over limit
                        result=0
                        safe_restart "$container" "crash loop — $RESTART_COUNT restarts" || result=$?
                        case $result in
                            0) ((T2_RESTARTS++))
                               queue_notify "$container crash loop on $(hostname) — restarted" "critical" ;;
                            2) : ;; # Added to skip list
                            *) queue_notify "$container crash loop CRITICAL on $(hostname) — manual intervention needed" "critical" ;;
                        esac
                    else
                        warn "$container — restarted since last check (Docker count: $RESTART_COUNT)"
                        queue_notify "$container restarted on $(hostname) — Docker count: $RESTART_COUNT" "warning"
                    fi
                fi
            done <<< "$ALL_CONTAINERS"
        fi

        # ── Dead containers ───────────────────────────────────────────────────────────────────
        # Routes through safe_restart() — ensures restart loop protection applies
        if [[ "$WATCHDOG_RESTART_DEAD" == "true" ]]; then
            DEAD=$(timeout "$DOCKER_TIMEOUT" docker ps -a \
                --filter status=dead --format "{{.Names}}" 2>/dev/null)
            while IFS= read -r container; do
                [[ -z "$container" ]] && continue
                [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
                is_skipped "$container" && continue
                error "$container — dead"
                ((T2_WARNINGS++))
                if [[ "$DRY_RUN" == false ]]; then
                    timeout "$DOCKER_TIMEOUT" docker rm "$container" >/dev/null 2>&1
                fi
                result=0
                safe_restart "$container" "dead container" || result=$?
                if [[ $result -eq 0 ]]; then
                    ((T2_RESTARTS++))
                    queue_notify "$container was dead on $(hostname) — removed and restarted" "warning"
                fi
            done <<< "$DEAD"
        fi

        # ── Unexpected exits ──────────────────────────────────────────────────────────────────
        # Only non-zero exit codes — exit 0 is a clean stop, not a crash
        # Skips containers already covered by WATCHDOG_REQUIRED_CONTAINERS (handled in Tier 1)
        if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then
            CRASHED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \
                --filter status=exited \
                --format "{{.Names}}|{{.Status}}" 2>/dev/null | \
                grep -v "Exited (0)")
            while IFS='|' read -r container status; do
                [[ -z "$container" ]] && continue
                [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
                is_skipped "$container" && continue
                # Skip containers already monitored by required containers (Tier 1)
                already_required=false
                for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]:-}"; do
                    [[ "$container" == "$req" ]] && already_required=true && break
                done
                [[ "$already_required" == true ]] && continue
                error "$container — $status (unexpected exit)"
                ((T2_WARNINGS++))
                result=0
                safe_restart "$container" "unexpected exit: $status" || result=$?
                if [[ $result -eq 0 ]]; then
                    ((T2_RESTARTS++))
                    queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning"
                fi
            done <<< "$CRASHED"
        fi

    fi # WATCHDOG_SCAN_ALL


    # ── Send notifications ────────────────────────────────────────────────────────────────────
    flush_notify

    # ── Cycle summary — quiet when healthy ───────────────────────────────────────────────────
    TOTAL_RESTARTS=$(( T1_RESTARTS + T2_RESTARTS ))
    TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS ))
    CYCLE_END=$(date +%s)

    CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l | tr -d ' ')
    if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then
        echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings   T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings"
        echo "$ICON_TIME      Duration: $(format_duration $(( CYCLE_END - CYCLE_START )))   Containers: $CONTAINER_COUNT"
        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    else
        echo "All healthy ✅ — ${CONTAINER_COUNT} containers ($(date '+%H:%M:%S'))"
    fi