Varaverk/Monitors/continuous_scripts_status.sh

#!/bin/bash
# ==============================================================================================
# ========================= Continuous Scripts Status ==========================================
# ==============================================================================================
# Live status dashboard for all continuously running scripts in the ecosystem.
# Run manually anytime — no schedule, no cron.
#
# ── WHAT IT SHOWS ─────────────────────────────────────────────────────────────────────────────
# For each continuous script (system_watchdog, docker_watchdog, failover):
#   Running state, PID, uptime, approximate cycle count
#   Active strikes and skip list
#   Recent restart history
#   Live health snapshot
#
# system_watchdog — rootfs, RAM, ZFS ARC, load, zombie count, CPU temp
# docker_watchdog — running/stopped/unhealthy containers, required containers,
#                   monitored memory containers, recent restart history
# failover        — current state, tier status, remote Tailscale visibility
#
# If a script is mid-cycle, state files are read as-is — reflects last completed cycle.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays.
# Required containers, tier delays, and Tailscale checks use MY_ID/REMOTE_ID correctly.
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   continuous_scripts_status.sh         — show dashboard
#   continuous_scripts_status.sh --log   — verbose output
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

# Dashboard script — output is the point
SILENT_MODE=false

parse_args "$@"

DOCKER_TIMEOUT=15

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays
detect_hosts

# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================

get_lock_pid() {
    local script_name="$1"
    local lockfile="$LOCK_DIR/${script_name}.lock"
    if [[ -f "$lockfile" ]]; then
        local content
        content=$(cat "$lockfile" 2>/dev/null)
        echo "${content%%:*}"
    fi
}

get_lock_name() {
    local script_name="$1"
    local lockfile="$LOCK_DIR/${script_name}.lock"
    if [[ -f "$lockfile" ]]; then
        local content
        content=$(cat "$lockfile" 2>/dev/null)
        echo "${content##*:}"
    fi
}

is_script_running() {
    local script_name="$1"
    local pid locked_name
    pid=$(get_lock_pid "$script_name")
    locked_name=$(get_lock_name "$script_name")
    [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$locked_name" == "$script_name" ]]
}

get_lock_age() {
    local script_name="$1"
    local lockfile="$LOCK_DIR/${script_name}.lock"
    if [[ -f "$lockfile" ]]; then
        local mtime now
        mtime=$(stat -c %Y "$lockfile" 2>/dev/null || echo 0)
        now=$(date +%s)
        echo $(( now - mtime ))
    else
        echo 0
    fi
}

# Human readable uptime — days/hours/mins
format_uptime() {
    local seconds=$1
    local days=$(( seconds / 86400 ))
    local hours=$(( (seconds % 86400) / 3600 ))
    local mins=$(( (seconds % 3600) / 60 ))
    if (( days > 0 )); then
        echo "${days}d ${hours}h ${mins}m"
    elif (( hours > 0 )); then
        echo "${hours}h ${mins}m"
    else
        echo "${mins}m"
    fi
}

divider() { printf '%.0s─' {1..57}; echo; }
section()  { echo ""; echo "  $1"; divider; }

# ==============================================================================================
# ━━━ Header ━━━
# ==============================================================================================
clear
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  🛡️  WATCHDOG STATUS — $(date '+%A, %B %-d at %-I:%M%p')"
echo "  $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME"
echo "  $ICON_HOST Remote: $REMOTE_ID — $REMOTE_SERVER_NAME"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

# ==============================================================================================
# ━━━ System Watchdog ━━━
# ==============================================================================================
section "⚙️  SYSTEM WATCHDOG"

SYS_PID=$(get_lock_pid "system_watchdog")
SYS_RUNNING=false

if is_script_running "system_watchdog"; then
    SYS_RUNNING=true
    SYS_AGE=$(get_lock_age "system_watchdog")
    SYS_UPTIME=$(format_uptime "$SYS_AGE")
    SYS_CYCLE=$(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL ))
    echo "  ✅ Running  │ PID: $SYS_PID │ Uptime: $SYS_UPTIME │ ~Cycle: $SYS_CYCLE"
    echo "  ⏱️  Interval: ${SYSTEM_WATCHDOG_INTERVAL}s │ Heartbeat every: ${SYSTEM_WATCHDOG_HEARTBEAT_HOURS}hr"
else
    echo "  ❌ NOT RUNNING — system_watchdog.sh is not active"
    echo "     Start via: bash Orchestrators/array_start.sh"
fi

echo ""

# System strikes
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
    ACTIVE_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
    if [[ -n "$ACTIVE_STRIKES" ]]; then
        echo "  ⚠️  Active strikes:"
        while IFS=: read -r key count; do
            [[ -z "$key" ]] && continue
            echo "     → $key: $count/$SYS_WATCHDOG_STRIKE_LIMIT"
        done <<< "$ACTIVE_STRIKES"
    else
        echo "  ✅ Strikes: none"
    fi
else
    echo "  ℹ️  Strike state file not found (watchdog may not have run yet)"
fi

# Reboot log
if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
    TOTAL_REBOOTS=$(grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0)
    TOTAL_REBOOTS="${TOTAL_REBOOTS//[^0-9]/}"
    TOTAL_REBOOTS="${TOTAL_REBOOTS:-0}"
    WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
    WEEK_REBOOTS=$(awk -v cutoff="$WEEK_CUTOFF" '$0 >= cutoff' \
        "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
    echo "  🔄 Watchdog reboots: $WEEK_REBOOTS this week / $TOTAL_REBOOTS total"
fi

# Container skip list
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
    SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
    echo ""
    echo "  ⛔ Skip list ($SKIP_COUNT — manual intervention needed):"
    while IFS= read -r container; do
        [[ -z "$container" ]] && continue
        echo "     → $container"
    done < "$SYS_WATCHDOG_FAILED_FILE"
else
    echo "  ✅ Skip list: empty"
fi

# Live system health snapshot
echo ""
echo "  📊 Current system state:"

ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
[[ "${ROOTFS_PCT:-0}" -ge "${SYS_WATCHDOG_ROOTFS_PCT:-95}" ]] && \
    ROOTFS_ICON="⚠️ " || ROOTFS_ICON="✅"
echo "     ${ROOTFS_ICON} rootfs: ${ROOTFS_PCT}% (threshold: ${SYS_WATCHDOG_ROOTFS_PCT}%)"

MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}")
MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
[[ $(printf "%.0f" "$MEM_FREE_GB") -lt "${SYS_WATCHDOG_MEM_GB:-4}" ]] && \
    MEM_ICON="⚠️ " || MEM_ICON="✅"
echo "     ${MEM_ICON} RAM: ${MEM_FREE_GB}GB free / ${MEM_TOTAL_GB}GB total (threshold: ${SYS_WATCHDOG_MEM_GB}GB free)"

if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
    ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
    ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
    ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX ))
    ARC_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}")
    [[ "$ARC_PCT" -ge "${SYS_WATCHDOG_ARC_PINNED_PCT:-98}" ]] && \
        ARC_ICON="⚠️ " || ARC_ICON="✅"
    echo "     ${ARC_ICON} ZFS ARC: ${ARC_GB}GB (${ARC_PCT}% of max, threshold: ${SYS_WATCHDOG_ARC_PINNED_PCT}%)"
fi

LOAD=$(awk '{print $1}' /proc/loadavg)
CORES=$(nproc)
LOAD_THRESH=$(( CORES * ${SYS_WATCHDOG_LOAD_MULTIPLIER:-3} ))
LOAD_INT=$(printf "%.0f" "$LOAD")
[[ "$LOAD_INT" -ge "$LOAD_THRESH" ]] && LOAD_ICON="⚠️ " || LOAD_ICON="✅"
echo "     ${LOAD_ICON} Load avg: $LOAD (threshold: ${LOAD_THRESH} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}x ${CORES} cores)"

ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0)
ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"
ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}"
[[ "$ZOMBIE_COUNT" -ge "${SYS_WATCHDOG_ZOMBIE_LIMIT:-50}" ]] && \
    ZOMBIE_ICON="⚠️ " || ZOMBIE_ICON="✅"
echo "     ${ZOMBIE_ICON} Zombies: $ZOMBIE_COUNT (threshold: ${SYS_WATCHDOG_ZOMBIE_LIMIT})"

if command -v sensors >/dev/null 2>&1; then
    CPU_TEMP=$(sensors 2>/dev/null | \
        grep -i "Package id 0\|Tctl\|CPU Temp" | \
        awk '{print $NF}' | tr -d '+°C' | head -1)
    if [[ -n "$CPU_TEMP" ]]; then
        CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP")
        [[ "$CPU_TEMP_INT" -ge "${SYS_WATCHDOG_CPU_TEMP_MAX:-95}" ]] && \
            TEMP_ICON="⚠️ " || TEMP_ICON="✅"
        echo "     ${TEMP_ICON} CPU temp: ${CPU_TEMP_INT}°C (threshold: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C)"
    fi
fi

# ==============================================================================================
# ━━━ Docker Watchdog ━━━
# ==============================================================================================
section "🐳  DOCKER WATCHDOG"

DOCKER_PID=$(get_lock_pid "docker_watchdog")
DOCKER_RUNNING=false

if is_script_running "docker_watchdog"; then
    DOCKER_RUNNING=true
    DOCKER_AGE=$(get_lock_age "docker_watchdog")
    DOCKER_UPTIME=$(format_uptime "$DOCKER_AGE")
    DOCKER_CYCLE=$(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL ))
    echo "  ✅ Running  │ PID: $DOCKER_PID │ Uptime: $DOCKER_UPTIME │ ~Cycle: $DOCKER_CYCLE"
    echo "  ⏱️  Interval: ${DOCKER_WATCHDOG_INTERVAL}s │ Heartbeat every: ${DOCKER_WATCHDOG_HEARTBEAT_HOURS}hr"
else
    echo "  ❌ NOT RUNNING — docker_watchdog.sh is not active"
    echo "     Start via: bash Orchestrators/array_start.sh"
fi

echo ""

# Container strikes
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
    ACTIVE_CONTAINER_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
    if [[ -n "$ACTIVE_CONTAINER_STRIKES" ]]; then
        echo "  ⚠️  Active container strikes:"
        while IFS=: read -r key count; do
            [[ -z "$key" ]] && continue
            echo "     → $key: $count"
        done <<< "$ACTIVE_CONTAINER_STRIKES"
    else
        echo "  ✅ Container strikes: none"
    fi
fi

# Container restart history
if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
    WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
    WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
        '$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l)
    if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then
        echo ""
        echo "  🔄 Container restarts this week: $WEEK_RESTARTS"
        awk -F'|' -v cutoff="$WEEK_CUTOFF" \
            '$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
            sort | uniq -c | sort -rn | head -5 | \
            while read -r count name; do
                echo "     → $name: $count restart(s)"
            done
    else
        echo "  ✅ Container restarts this week: none"
    fi
fi

# Container overview
echo ""
echo "  📦 Container overview:"

if command -v docker >/dev/null 2>&1; then
    RUNNING=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l)
    TOTAL=$(timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l)
    UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \
        --filter health=unhealthy -q 2>/dev/null | wc -l)

    # Stopped containers — filter intentionally ignored ones
    STOPPED_FILTERED=()
    while IFS= read -r name; do
        [[ -z "$name" ]] && continue
        SKIP=false
        for ignore in "${WATCHDOG_SCAN_IGNORE[@]}"; do
            [[ "$name" == "$ignore" ]] && SKIP=true && break
        done
        [[ "$SKIP" == false ]] && STOPPED_FILTERED+=("$name")
    done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \
        --format "{{.Names}}" 2>/dev/null)

    STOPPED_COUNT="${#STOPPED_FILTERED[@]}"
    echo "     Running: $RUNNING / $TOTAL total"
    [[ "$UNHEALTHY" -gt 0 ]] && echo "     ⚠️  Unhealthy: $UNHEALTHY"

    if [[ "$STOPPED_COUNT" -gt 0 ]]; then
        echo "     ⚠️  Stopped (unexpected):"
        for name in "${STOPPED_FILTERED[@]}"; do
            echo "        → $name"
        done
    else
        echo "     ✅ All containers running"
    fi

    # Required containers — aliased by detect_hosts() → WATCHDOG_REQUIRED_CONTAINERS
    REQUIRED_ISSUES=0
    if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
        echo ""
        echo "  🔐 Required containers:"
        for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
            [[ -z "$container" ]] && continue
            STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
                '{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
            if [[ "$STATUS" == "true" ]]; then
                echo "     ✅ $container"
            else
                echo "     ❌ $container — $STATUS"
                (( REQUIRED_ISSUES++ ))
            fi
        done
    fi

    # Memory-monitored containers — aliased by detect_hosts() → WATCHDOG_CONTAINERS
    if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
        echo ""
        echo "  📊 Monitored containers (memory):"
        for container in "${!WATCHDOG_CONTAINERS[@]}"; do
            LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
            LIMIT_GB=$(awk "BEGIN {printf \"%.0f\", $LIMIT_MB / 1024}")
            USAGE=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
                --format "{{.MemUsage}}" "$container" 2>/dev/null | awk '{print $1}')
            STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
                '{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
            if [[ "$STATUS" == "true" ]]; then
                echo "     ✅ $container: ${USAGE:-?} (limit: ${LIMIT_GB}GB)"
            else
                echo "     ❌ $container: not running (limit: ${LIMIT_GB}GB)"
            fi
        done
    fi
else
    echo "     Docker not available"
fi

# ==============================================================================================
# ━━━ Failover ━━━
# ==============================================================================================
section "🔀  FAILOVER"

FAILOVER_PID=$(get_lock_pid "failover")
FAILOVER_RUNNING=false

if is_script_running "failover"; then
    FAILOVER_RUNNING=true
    FAILOVER_AGE=$(get_lock_age "failover")
    FAILOVER_UPTIME=$(format_uptime "$FAILOVER_AGE")
    echo "  ✅ Running  │ PID: $FAILOVER_PID │ Uptime: $FAILOVER_UPTIME"
else
    if [[ "${FAILOVER_ENABLED:-true}" == false ]]; then
        echo "  ⏸️  Disabled — FAILOVER_ENABLED=false in master.conf"
    else
        echo "  ❌ NOT RUNNING — failover.sh is not active"
        echo "     Start via: bash Orchestrators/array_start.sh"
    fi
fi

echo ""

# Failover state
FAILOVER_STATE="UNKNOWN"
FAILOVER_STATE_SECONDS=0

if [[ -f "$FAILOVER_STATE_FILE" ]]; then
    FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
    FAILOVER_LAST_EPOCH=$(grep "^failover_start=" "$FAILOVER_STATE_FILE" \
        2>/dev/null | cut -d= -f2)
    if [[ -n "$FAILOVER_LAST_EPOCH" && "$FAILOVER_LAST_EPOCH" -gt 0 ]]; then
        FAILOVER_STATE_SECONDS=$(( $(date +%s) - FAILOVER_LAST_EPOCH ))
    fi
fi

STATE_DURATION=$(format_uptime "${FAILOVER_STATE_SECONDS:-0}")

# Tier delays via REMOTE_ID — same logic as failover.sh
REMOTE_TIER2_VAR="${REMOTE_ID}_TIER2_DELAY"
REMOTE_TIER3_VAR="${REMOTE_ID}_TIER3_DELAY"
REMOTE_TIER4_VAR="${REMOTE_ID}_TIER4_DELAY"
TIER2_DELAY="${!REMOTE_TIER2_VAR:-240}"
TIER3_DELAY="${!REMOTE_TIER3_VAR:-720}"
TIER4_DELAY="${!REMOTE_TIER4_VAR:-1440}"

case "$FAILOVER_STATE" in
    NORMAL)
        echo "  ✅ State: NORMAL"
        ;;
    FAILOVER)
        echo "  ⚠️  State: FAILOVER — $REMOTE_SERVER_NAME is down"
        echo "  ⏱️  Duration: $STATE_DURATION"
        FAILOVER_MINS=$(( FAILOVER_STATE_SECONDS / 60 ))
        echo ""
        echo "  🔄 Tier status:"
        echo "     Tier 1 (immediate):       ✅ active"
        if (( FAILOVER_MINS >= TIER2_DELAY )); then
            echo "     Tier 2 (${TIER2_DELAY}min):  ✅ active"
        else
            REMAINING=$(( TIER2_DELAY - FAILOVER_MINS ))
            echo "     Tier 2 (${TIER2_DELAY}min):  ⏳ in ${REMAINING}min"
        fi
        if (( FAILOVER_MINS >= TIER3_DELAY )); then
            echo "     Tier 3 (${TIER3_DELAY}min): ✅ active"
        else
            REMAINING=$(( TIER3_DELAY - FAILOVER_MINS ))
            echo "     Tier 3 (${TIER3_DELAY}min): ⏳ in ${REMAINING}min"
        fi
        if (( FAILOVER_MINS >= TIER4_DELAY )); then
            echo "     Tier 4 (${TIER4_DELAY}min): ✅ active"
        else
            REMAINING=$(( TIER4_DELAY - FAILOVER_MINS ))
            echo "     Tier 4 (${TIER4_DELAY}min): ⏳ in ${REMAINING}min"
        fi
        ;;
    NO_INTERNET)
        echo "  ❌ State: NO_INTERNET — DDNS stopped"
        echo "  ⏱️  Down for: $STATE_DURATION"
        ;;
    DARK)
        echo "  ❌ State: DARK — $REMOTE_SERVER_NAME down AND no internet"
        echo "  ⏱️  Duration: $STATE_DURATION"
        ;;
    *)
        echo "  ❓ State: ${FAILOVER_STATE:-unknown}"
        ;;
esac

# Tailscale remote visibility — uses REMOTE_SERVER_NAME from detect_hosts()
echo ""
if command -v tailscale >/dev/null 2>&1; then
    REMOTE_IP=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null)
    if [[ -n "$REMOTE_IP" ]]; then
        if ping -c 1 -W 2 "$REMOTE_IP" >/dev/null 2>&1; then
            echo "  🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_IP — reachable ✅"
        else
            echo "  🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_IP — not responding ⚠️"
        fi
    else
        echo "  🌐 $REMOTE_ID ($REMOTE_SERVER_NAME) not visible on Tailscale ❌"
    fi
else
    echo "  🌐 Tailscale: not available"
fi

echo "  📡 Check interval: ${FAILOVER_CHECK_INTERVAL}s │ Handback strikes: ${FAILOVER_HANDBACK_STRIKES}"

# ==============================================================================================
# ━━━ Footer ━━━
# ==============================================================================================
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

ISSUES=0
[[ "$SYS_RUNNING" == false ]] && (( ISSUES++ ))
[[ "$DOCKER_RUNNING" == false ]] && (( ISSUES++ ))
[[ "$FAILOVER_RUNNING" == false && "${FAILOVER_ENABLED:-true}" != false ]] && (( ISSUES++ ))
[[ -n "$ACTIVE_STRIKES" ]] && (( ISSUES++ ))
[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && (( ISSUES++ ))
[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && (( ISSUES++ ))
[[ "$FAILOVER_STATE" != "NORMAL" && "$FAILOVER_STATE" != "UNKNOWN" ]] && (( ISSUES++ ))

if [[ "$ISSUES" -eq 0 ]]; then
    echo "  ✅ $MY_ID — all continuous scripts healthy"
else
    echo "  ⚠️  $ISSUES issue(s) detected — review above"
fi

echo "  🕐 Checked: $(date '+%H:%M:%S')"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""