#!/bin/bash
# -----------------------------------------------------------------------------------------------
# ----------------------------- Continuous Scripts Status --------------------------------------
# -----------------------------------------------------------------------------------------------
# Live status dashboard for all continuously running scripts in the ecosystem.
# Run manually anytime — no schedule, no cron.
#
# Covers all scripts started by array_start.sh that run until array stops:
#   system_watchdog.sh   — system health monitor
#   docker_watchdog.sh   — container health monitor
#   failover.sh          — mutual failover monitor
#
# Shows for each:
#   Running state, PID, uptime, current cycle
#   Active strikes, skip list, recent actions
#   Live system/container health snapshot
#   Failover state, tier status, Tailscale connectivity
#
# If a script is mid-cycle state files are read as-is — reflects last completed cycle.
# Run: bash Monitors/continuous_scripts_status.sh
# -----------------------------------------------------------------------------------------------

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"

parse_args "$@"

# -----------------------------------------------------------------------------------------------
# HELPERS
# -----------------------------------------------------------------------------------------------

get_lock_pid() {
    local script_name="$1"
    local lockfile="$LOCK_DIR/${script_name}.lock"
    if [[ -f "$lockfile" ]]; then
        local content
        content=$(cat "$lockfile" 2>/dev/null)
        echo "${content%%:*}"
    fi
}

get_lock_name() {
    local script_name="$1"
    local lockfile="$LOCK_DIR/${script_name}.lock"
    if [[ -f "$lockfile" ]]; then
        local content
        content=$(cat "$lockfile" 2>/dev/null)
        echo "${content##*:}"
    fi
}

is_watchdog_running() {
    local script_name="$1"
    local pid
    pid=$(get_lock_pid "$script_name")
    local locked_name
    locked_name=$(get_lock_name "$script_name")
    [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$locked_name" == "$script_name" ]]
}

get_lock_age() {
    local script_name="$1"
    local lockfile="$LOCK_DIR/${script_name}.lock"
    if [[ -f "$lockfile" ]]; then
        local mtime now
        mtime=$(stat -c %Y "$lockfile" 2>/dev/null || echo 0)
        now=$(date +%s)
        echo $(( now - mtime ))
    else
        echo 0
    fi
}

format_uptime() {
    local seconds=$1
    local days=$(( seconds / 86400 ))
    local hours=$(( (seconds % 86400) / 3600 ))
    local mins=$(( (seconds % 3600) / 60 ))
    if (( days > 0 )); then
        echo "${days}d ${hours}h ${mins}m"
    elif (( hours > 0 )); then
        echo "${hours}h ${mins}m"
    else
        echo "${mins}m"
    fi
}

get_strikes() {
    local state_file="$1"
    local key="$2"
    grep -E "^${key}:" "$state_file" 2>/dev/null | cut -d: -f2
}

divider() { printf '%.0s─' {1..55}; echo; }
header() { echo ""; echo "  $1"; divider; }

# -----------------------------------------------------------------------------------------------
# ━━━ HEADER ━━━
# -----------------------------------------------------------------------------------------------
clear
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  🛡️  WATCHDOG STATUS — $(date '+%A, %B %-d at %-I:%M%p')"
echo "  🖥️  $(hostname)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

# -----------------------------------------------------------------------------------------------
# ━━━ System Watchdog ━━━
# -----------------------------------------------------------------------------------------------
header "⚙️  SYSTEM WATCHDOG"

SYS_PID=$(get_lock_pid "system_watchdog")
SYS_RUNNING=false

if is_watchdog_running "system_watchdog"; then
    SYS_RUNNING=true
    SYS_AGE=$(get_lock_age "system_watchdog")
    SYS_UPTIME=$(format_uptime "$SYS_AGE")
    SYS_CYCLE=$(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL ))
    echo "  ✅ Running  │ PID: $SYS_PID │ Uptime: $SYS_UPTIME │ ~Cycle: $SYS_CYCLE"
    echo "  ⏱️  Interval: ${SYSTEM_WATCHDOG_INTERVAL}s │ Heartbeat every: ${SYSTEM_WATCHDOG_HEARTBEAT_HOURS}hr"
else
    echo "  ❌ NOT RUNNING — system_watchdog.sh is not active"
    echo "     Start via: bash Orchestrators/array_start.sh"
fi

echo ""

# System watchdog strikes
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
    ACTIVE_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
    if [[ -n "$ACTIVE_STRIKES" ]]; then
        echo "  ⚠️  Active strikes:"
        while IFS=: read -r key count; do
            [[ -z "$key" ]] && continue
            echo "     → $key: $count/$SYS_WATCHDOG_STRIKE_LIMIT"
        done <<< "$ACTIVE_STRIKES"
    else
        echo "  ✅ Strikes: none"
    fi
else
    echo "  ℹ️  Strike state file not found (watchdog may not have run yet)"
fi

# Reboot log
if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
    TOTAL_REBOOTS=$(grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0)
    WEEK_EPOCH=$(date -d "7 days ago" +%s)
    WEEK_REBOOTS=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \
        "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
    echo "  🔄 Watchdog reboots: $WEEK_REBOOTS this week / $TOTAL_REBOOTS total"
fi

# Container skip list
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
    SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
    echo ""
    echo "  ⛔ Skip list ($SKIP_COUNT containers — manual intervention needed):"
    while IFS= read -r container; do
        [[ -z "$container" ]] && continue
        echo "     → $container"
    done < "$SYS_WATCHDOG_FAILED_FILE"
else
    echo "  ✅ Skip list: empty"
fi

# Current system health snapshot
echo ""
echo "  📊 Current system state:"

# rootfs
ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
[[ "${ROOTFS_PCT:-0}" -ge "${SYS_WATCHDOG_ROOTFS_PCT:-95}" ]] && \
    ROOTFS_ICON="⚠️ " || ROOTFS_ICON="✅"
echo "     ${ROOTFS_ICON} rootfs: ${ROOTFS_PCT}% (threshold: ${SYS_WATCHDOG_ROOTFS_PCT}%)"

# RAM
MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}")
MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
[[ $(printf "%.0f" "$MEM_FREE_GB") -lt "${SYS_WATCHDOG_MEM_GB:-4}" ]] && \
    MEM_ICON="⚠️ " || MEM_ICON="✅"
echo "     ${MEM_ICON} RAM: ${MEM_FREE_GB}GB free / ${MEM_TOTAL_GB}GB total (threshold: ${SYS_WATCHDOG_MEM_GB}GB free)"

# ARC
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
    ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
    ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
    ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX ))
    ARC_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}")
    [[ "$ARC_PCT" -ge "${SYS_WATCHDOG_ARC_PINNED_PCT:-98}" ]] && \
        ARC_ICON="⚠️ " || ARC_ICON="✅"
    echo "     ${ARC_ICON} ZFS ARC: ${ARC_GB}GB (${ARC_PCT}% of max, threshold: ${SYS_WATCHDOG_ARC_PINNED_PCT}%)"
fi

# Load
LOAD=$(awk '{print $1}' /proc/loadavg)
CORES=$(nproc)
LOAD_THRESH=$(( CORES * ${SYS_WATCHDOG_LOAD_MULTIPLIER:-3} ))
LOAD_INT=$(printf "%.0f" "$LOAD")
[[ "$LOAD_INT" -ge "$LOAD_THRESH" ]] && LOAD_ICON="⚠️ " || LOAD_ICON="✅"
echo "     ${LOAD_ICON} Load avg: $LOAD (threshold: ${LOAD_THRESH} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}x ${CORES} cores)"

# Zombies
ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0)
ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"
ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}"
[[ "$ZOMBIE_COUNT" -ge "${SYS_WATCHDOG_ZOMBIE_LIMIT:-50}" ]] && \
    ZOMBIE_ICON="⚠️ " || ZOMBIE_ICON="✅"
echo "     ${ZOMBIE_ICON} Zombies: $ZOMBIE_COUNT (threshold: ${SYS_WATCHDOG_ZOMBIE_LIMIT})"

# CPU temp
if command -v sensors >/dev/null 2>&1; then
    CPU_TEMP=$(sensors 2>/dev/null | \
        grep -i "Package id 0\|Tctl\|CPU Temp" | \
        awk '{print $NF}' | tr -d '+°C' | head -1)
    if [[ -n "$CPU_TEMP" ]]; then
        CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP")
        [[ "$CPU_TEMP_INT" -ge "${SYS_WATCHDOG_CPU_TEMP_MAX:-95}" ]] && \
            TEMP_ICON="⚠️ " || TEMP_ICON="✅"
        echo "     ${TEMP_ICON} CPU temp: ${CPU_TEMP_INT}°C (threshold: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C)"
    fi
fi

# -----------------------------------------------------------------------------------------------
# ━━━ Docker Watchdog ━━━
# -----------------------------------------------------------------------------------------------
header "🐳  DOCKER WATCHDOG"

DOCKER_PID=$(get_lock_pid "docker_watchdog")
DOCKER_RUNNING=false

if is_watchdog_running "docker_watchdog"; then
    DOCKER_RUNNING=true
    DOCKER_AGE=$(get_lock_age "docker_watchdog")
    DOCKER_UPTIME=$(format_uptime "$DOCKER_AGE")
    DOCKER_CYCLE=$(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL ))
    echo "  ✅ Running  │ PID: $DOCKER_PID │ Uptime: $DOCKER_UPTIME │ ~Cycle: $DOCKER_CYCLE"
    echo "  ⏱️  Interval: ${DOCKER_WATCHDOG_INTERVAL}s │ Heartbeat every: ${DOCKER_WATCHDOG_HEARTBEAT_HOURS}hr"
else
    echo "  ❌ NOT RUNNING — docker_watchdog.sh is not active"
    echo "     Start via: bash Orchestrators/array_start.sh"
fi

echo ""

# Container watchdog strikes
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
    ACTIVE_CONTAINER_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
    if [[ -n "$ACTIVE_CONTAINER_STRIKES" ]]; then
        echo "  ⚠️  Active container strikes:"
        while IFS=: read -r key count; do
            [[ -z "$key" ]] && continue
            echo "     → $key: $count"
        done <<< "$ACTIVE_CONTAINER_STRIKES"
    else
        echo "  ✅ Container strikes: none"
    fi
fi

# Container restart history this week
if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
    WEEK_EPOCH=$(date -d "7 days ago" +%s 2>/dev/null || date -v-7d +%s 2>/dev/null)
    WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_EPOCH" \
        'NR>0 {if ($2 >= cutoff) count++} END {print count+0}' \
        "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null)
    if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then
        echo ""
        echo "  🔄 Container restarts this week: $WEEK_RESTARTS"
        awk -F'|' -v cutoff="$WEEK_EPOCH" \
            '$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
            sort | uniq -c | sort -rn | head -5 | \
            while read -r count name; do
                echo "     → $name: $count restart(s)"
            done
    else
        echo "  ✅ Container restarts this week: none"
    fi
fi

# Docker container overview
echo ""
echo "  📦 Container overview:"

if command -v docker >/dev/null 2>&1; then
    RUNNING=$(docker ps -q 2>/dev/null | wc -l)
    TOTAL=$(docker ps -aq 2>/dev/null | wc -l)
    UNHEALTHY=$(docker ps --filter health=unhealthy -q 2>/dev/null | wc -l)
    STOPPED=$(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null)
    STOPPED_COUNT=$(echo "$STOPPED" | grep -c . 2>/dev/null || echo 0)

    echo "     Running: $RUNNING / $TOTAL total"
    [[ "$UNHEALTHY" -gt 0 ]] && echo "     ⚠️  Unhealthy: $UNHEALTHY"
    if [[ "$STOPPED_COUNT" -gt 0 ]]; then
        echo "     ⚠️  Stopped containers:"
        echo "$STOPPED" | head -10 | while IFS= read -r name; do
            [[ -z "$name" ]] && continue
            echo "        → $name"
        done
    else
        echo "     ✅ All containers running"
    fi

    # Check required containers
    detect_hosts 2>/dev/null
    if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
        REQUIRED=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}")
    else
        REQUIRED=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}")
    fi

    REQUIRED_ISSUES=0
    if [[ ${#REQUIRED[@]} -gt 0 ]]; then
        echo ""
        echo "  🔐 Required containers:"
        for container in "${REQUIRED[@]}"; do
            [[ -z "$container" ]] && continue
            STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
            if [[ "$STATUS" == "true" ]]; then
                echo "     ✅ $container"
            else
                echo "     ❌ $container — $STATUS"
                ((REQUIRED_ISSUES++))
            fi
        done
    fi

    # Tier 1 monitored containers from WATCHDOG_CONTAINERS
    if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
        echo ""
        echo "  📊 Monitored containers (memory):"
        for container in "${!WATCHDOG_CONTAINERS[@]}"; do
            LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
            LIMIT_GB=$(awk "BEGIN {printf \"%.0f\", $LIMIT_MB / 1024}")
            USAGE=$(docker stats --no-stream --format "{{.MemUsage}}" "$container" \
                2>/dev/null | awk '{print $1}')
            STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
            if [[ "$STATUS" == "true" ]]; then
                echo "     ✅ $container: ${USAGE:-?} (limit: ${LIMIT_GB}GB)"
            else
                echo "     ❌ $container: not running (limit: ${LIMIT_GB}GB)"
            fi
        done
    fi
else
    echo "     Docker not available"
fi

# -----------------------------------------------------------------------------------------------
# ━━━ Failover ━━━
# -----------------------------------------------------------------------------------------------
header "🔀  FAILOVER"

FAILOVER_PID=$(get_lock_pid "failover")
FAILOVER_RUNNING=false

if is_watchdog_running "failover"; then
    FAILOVER_RUNNING=true
    FAILOVER_AGE=$(get_lock_age "failover")
    FAILOVER_UPTIME=$(format_uptime "$FAILOVER_AGE")
    echo "  ✅ Running  │ PID: $FAILOVER_PID │ Uptime: $FAILOVER_UPTIME"
else
    echo "  ❌ NOT RUNNING — failover.sh is not active"
    echo "     Start via: bash Orchestrators/array_start.sh"
fi

echo ""

# Failover state
FAILOVER_STATE="UNKNOWN"
FAILOVER_LAST_CHANGE=""
FAILOVER_STATE_SECONDS=0

if [[ -f "$FAILOVER_STATE_FILE" ]]; then
    FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
    FAILOVER_LAST_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
    FAILOVER_LAST_EPOCH=$(grep "^last_change_epoch=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
    if [[ -n "$FAILOVER_LAST_EPOCH" ]]; then
        FAILOVER_STATE_SECONDS=$(( $(date +%s) - FAILOVER_LAST_EPOCH ))
    fi
fi

STATE_DURATION=$(format_uptime "${FAILOVER_STATE_SECONDS:-0}")

case "$FAILOVER_STATE" in
    NORMAL)
        echo "  ✅ State: NORMAL"
        echo "  📅 In NORMAL state for: $STATE_DURATION"
        ;;
    FAILOVER)
        echo "  ⚠️  State: FAILOVER — remote server down"
        echo "  ⏱️  Duration: $STATE_DURATION"
        # Show which tiers are active
        TIER1_DELAY=0
        if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
            TIER2_DELAY=$HOST2_TIER2_DELAY
            TIER3_DELAY=$HOST2_TIER3_DELAY
            TIER4_DELAY=$HOST2_TIER4_DELAY
        else
            TIER2_DELAY=$HOST1_TIER2_DELAY
            TIER3_DELAY=$HOST1_TIER3_DELAY
            TIER4_DELAY=$HOST1_TIER4_DELAY
        fi
        FAILOVER_MINS=$(( FAILOVER_STATE_SECONDS / 60 ))
        echo ""
        echo "  🔄 Tier status:"
        echo "     Tier 1 (immediate):  ✅ active"
        if (( FAILOVER_MINS >= TIER2_DELAY )); then
            echo "     Tier 2 (${TIER2_DELAY}min):     ✅ active"
        else
            REMAINING=$(( TIER2_DELAY - FAILOVER_MINS ))
            echo "     Tier 2 (${TIER2_DELAY}min):     ⏳ activates in ${REMAINING}min"
        fi
        if (( FAILOVER_MINS >= TIER3_DELAY )); then
            echo "     Tier 3 (${TIER3_DELAY}min):    ✅ active"
        else
            REMAINING=$(( TIER3_DELAY - FAILOVER_MINS ))
            echo "     Tier 3 (${TIER3_DELAY}min):    ⏳ activates in ${REMAINING}min"
        fi
        if (( FAILOVER_MINS >= TIER4_DELAY )); then
            echo "     Tier 4 (${TIER4_DELAY}min):   ✅ active"
        else
            REMAINING=$(( TIER4_DELAY - FAILOVER_MINS ))
            echo "     Tier 4 (${TIER4_DELAY}min):   ⏳ activates in ${REMAINING}min"
        fi
        ;;
    NO_INTERNET)
        echo "  ❌ State: NO_INTERNET — DDNS stopped"
        echo "  ⏱️  Down for: $STATE_DURATION"
        ;;
    DARK)
        echo "  ❌ State: DARK — remote down AND no internet"
        echo "  ⏱️  Duration: $STATE_DURATION"
        ;;
    *)
        echo "  ❓ State: ${FAILOVER_STATE:-unknown}"
        ;;
esac

# Tailscale remote visibility
echo ""
if command -v tailscale >/dev/null 2>&1; then
    REMOTE_IP=$(tailscale ip -4 "$HOST2" 2>/dev/null)
    if [[ -n "$REMOTE_IP" ]]; then
        # Try a quick ping to see last seen
        if ping -c 1 -W 2 "$REMOTE_IP" >/dev/null 2>&1; then
            echo "  🌐 Remote: $REMOTE_IP │ reachable ✅"
        else
            echo "  🌐 Remote: $REMOTE_IP │ not responding ⚠️"
        fi
    else
        echo "  🌐 Remote: $HOST2 not visible on Tailscale ❌"
    fi
else
    echo "  🌐 Tailscale: not available"
fi

echo "  📡 Check interval: ${FAILOVER_CHECK_INTERVAL}s │ Handback strikes: ${FAILOVER_HANDBACK_STRIKES}"

# -----------------------------------------------------------------------------------------------
# ━━━ Footer ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

# Overall status
ISSUES=0
[[ "$SYS_RUNNING" == false ]] && ((ISSUES++))
[[ "$DOCKER_RUNNING" == false ]] && ((ISSUES++))
[[ "$FAILOVER_RUNNING" == false ]] && ((ISSUES++))
[[ -n "$ACTIVE_STRIKES" ]] && ((ISSUES++))
[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && ((ISSUES++))
[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && ((ISSUES++))
[[ "$FAILOVER_STATE" != "NORMAL" ]] && [[ "$FAILOVER_STATE" != "UNKNOWN" ]] && ((ISSUES++))

if [[ "$ISSUES" -eq 0 ]]; then
    echo "  ✅ All continuous scripts healthy — no issues detected"
else
    echo "  ⚠️  $ISSUES issue(s) detected — review above"
fi

echo "  🕐 Checked at: $(date '+%H:%M:%S')"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""