#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- System Watchdog --------------------------------------------
# -----------------------------------------------------------------------------------------------
# Last line of defense — reboots the system cleanly if it is about to become unstable.
# Runs continuously as a background process — started by array_start.sh at array start.
# Works alongside docker_watchdog.sh which handles container-level healing first.
#
# Checks (all toggleable in Master.conf):
#   rootfs usage      — high rootfs fills rapidly when array is down, crash imminent
#   /var/log usage    — log spam can fill rootfs, indicates something is broken
#   free RAM          — critically low RAM means OOM or swap imminent
#   ZFS ARC pinned    — ARC not releasing after reclaim means memory is stuck
#   CPU temperature   — sustained tjmax causes throttling or kernel panic
#   load average      — sustained high load means something is stuck or runaway
#   zombie processes  — large zombie count indicates serious process management failure
#   Docker daemon     — unresponsive daemon means containers cannot be managed
#   Required containers — stopped containers that should be running (after watchdog skip list)
#
# Abort conditions (toggleable):
#   ZFS pool unhealthy  — reboot with bad pool risks data loss
#   Parity running      — aborting parity is better than crashing mid-check
#   Mover running       — aborting move is better than crashing mid-move
#
# Reboot loop protection:
#   Tracks reboot timestamps in persistent log on /boot/
#   Rolling window — old entries purge automatically
#   If reboot count hits limit in window → shutdown instead of reboot
#
# Continuous loop:
#   Checks run every SYSTEM_WATCHDOG_INTERVAL seconds (default 300 = 5min)
#   Master.conf re-sourced each cycle — config changes picked up without restart
#   Silent when healthy — only verbose when trigger or reboot
#   Clean shutdown on SIGTERM/SIGINT — sent by array stop
#
# All configuration in Master.conf under System Watchdog section.
# Supports --dry-run to show triggered conditions without rebooting.
# -----------------------------------------------------------------------------------------------

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"

parse_args "$@"

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup — runs once at start ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

success "Running as root"

acquire_lock "continuous"

TOTAL_CORES=$(nproc)

# Ensure state and persistent files exist
touch "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || {
    error "Cannot create state file: $SYS_WATCHDOG_STATE_FILE"
    exit 1
}

touch "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || {
    error "Cannot create reboot log: $SYS_WATCHDOG_REBOOT_LOG"
    exit 1
}

touch "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || {
    error "Cannot create failed container list: $SYS_WATCHDOG_FAILED_FILE"
    exit 1
}

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HEALTH   rootfs threshold:    ${SYS_WATCHDOG_ROOTFS_PCT}%     toggle: $SYS_WATCHDOG_CHECK_ROOTFS"
    echo "$ICON_HEALTH   log threshold:       ${SYS_WATCHDOG_LOG_PCT}%     toggle: $SYS_WATCHDOG_CHECK_LOG"
    echo "$ICON_MEM      RAM threshold:       ${SYS_WATCHDOG_MEM_GB}GB free  toggle: $SYS_WATCHDOG_CHECK_RAM"
    echo "$ICON_ZFS      ARC pinned:          ${SYS_WATCHDOG_ARC_PINNED_PCT}%     toggle: $SYS_WATCHDOG_CHECK_ARC"
    echo "$ICON_ZFS      ARC release:         ${SYS_WATCHDOG_ARC_RELEASE_PCT}%"
    echo "$ICON_GEAR     CPU temp max:        ${SYS_WATCHDOG_CPU_TEMP_MAX}°C    toggle: $SYS_WATCHDOG_CHECK_CPU_TEMP"
    echo "$ICON_GEAR     Load multiplier:     ${SYS_WATCHDOG_LOAD_MULTIPLIER}x cores  toggle: $SYS_WATCHDOG_CHECK_LOAD"
    echo "$ICON_GEAR     Zombie limit:        ${SYS_WATCHDOG_ZOMBIE_LIMIT}       toggle: $SYS_WATCHDOG_CHECK_ZOMBIES"
    echo "$ICON_CONTAINERS Docker daemon:     toggle: $SYS_WATCHDOG_CHECK_DOCKER_DAEMON"
    echo "$ICON_CONTAINERS Containers:        toggle: $SYS_WATCHDOG_CHECK_CONTAINERS"
    echo "$ICON_SHIELD   Strike limit:        $SYS_WATCHDOG_STRIKE_LIMIT"
    echo "$ICON_TIME     Interval:            ${SYSTEM_WATCHDOG_INTERVAL}s"
    echo "$ICON_REBOOT_SMART Reboot limit:   $SYS_WATCHDOG_REBOOT_LIMIT in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs"
    echo "$ICON_ZFS      Abort ZFS unhealthy: $SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY"
    echo "$ICON_GEAR     Abort on parity:     $SYS_WATCHDOG_ABORT_ON_PARITY"
    echo "$ICON_MOVER    Abort on mover:      $SYS_WATCHDOG_ABORT_ON_MOVER"
    echo "$ICON_GEAR     Dry Run:             $DRY_RUN"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboot will be executed"

# -----------------------------------------------------------------------------------------------
# STATE HELPERS — defined once, used every cycle
# -----------------------------------------------------------------------------------------------

get_strikes() {
    local key="$1"
    grep -E "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2
}

set_strikes() {
    local key="$1" count="$2"
    grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp"
    echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp"
    mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE"
}

increment_strikes() {
    local key="$1"
    local current
    current=$(get_strikes "$key")
    [[ -z "$current" ]] && current=0
    ((current++))
    set_strikes "$key" "$current"
    echo "$current"
}

reset_strikes() {
    local key="$1"
    set_strikes "$key" 0
}

purge_old_reboots() {
    local now
    now=$(date +%s)
    local cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW ))
    grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do
        [[ "$ts" -gt "$cutoff" ]] && echo "$ts"
    done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp"
    mv "${SYS_WATCHDOG_REBOOT_LOG}.tmp" "$SYS_WATCHDOG_REBOOT_LOG"
}

count_recent_reboots() {
    purge_old_reboots
    grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0
}

log_reboot() {
    date +%s >> "$SYS_WATCHDOG_REBOOT_LOG"
}

check_abort_conditions() {
    local should_abort=false

    if command -v zpool >/dev/null 2>&1; then
        local unhealthy
        unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true)
        if [[ -n "$unhealthy" ]]; then
            if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then
                error "$ICON_ZFS ZFS pool unhealthy — aborting reboot"
                notify "System watchdog aborted reboot on $(hostname) — ZFS pool unhealthy" "System Watchdog" "warning"
                should_abort=true
            else
                warn "$ICON_ZFS ZFS pool unhealthy — continuing reboot"
            fi
        fi
    fi

    if [[ -f /var/local/emhttp/parity-date.txt ]]; then
        if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then
            if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then
                error "$ICON_DISK Parity check running — aborting reboot"
                notify "System watchdog aborted reboot on $(hostname) — parity running" "System Watchdog" "warning"
                should_abort=true
            else
                warn "$ICON_DISK Parity check running — continuing reboot"
            fi
        fi
    fi

    if pgrep -f "mover" >/dev/null 2>&1; then
        if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then
            error "$ICON_MOVER Mover running — aborting reboot"
            notify "System watchdog aborted reboot on $(hostname) — mover running" "System Watchdog" "warning"
            should_abort=true
        else
            warn "$ICON_MOVER Mover running — continuing reboot"
        fi
    fi

    [[ "$should_abort" == true ]] && return 1
    return 0
}

run_strike_check() {
    local key="$1" triggered="$2" description="$3"
    if [[ "$triggered" == true ]]; then
        local strikes
        strikes=$(increment_strikes "$key")
        warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT"
        if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then
            error "$description hit strike limit — reboot triggered"
            reset_strikes "$key"
            return 0
        fi
    else
        local current
        current=$(get_strikes "$key")
        if [[ -n "$current" && "$current" -gt 0 ]]; then
            reset_strikes "$key"
        fi
    fi
    return 1
}

do_reboot() {
    local triggers=("$@")

    if ! check_abort_conditions; then
        return
    fi

    RECENT_REBOOTS=$(count_recent_reboots)
    info "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT"

    if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then
        error "Reboot limit hit — shutting down instead"
        notify "Reboot loop detected on $(hostname) — shutting down after $RECENT_REBOOTS reboots — conditions: ${triggers[*]}" "System Watchdog" "warning"
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would shutdown now"
            return
        fi
        sync
        /sbin/poweroff
        return
    fi

    notify "System watchdog reboot triggered on $(hostname) — conditions: ${triggers[*]}" "System Watchdog" "warning"

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — reboot sequence would begin now"
        return
    fi

    log_reboot

    info "Shutting down VMs..."
    if command -v virsh >/dev/null 2>&1; then
        for VM in $(virsh list --name 2>/dev/null); do
            [[ -z "$VM" ]] && continue
            virsh shutdown "$VM" >/dev/null 2>&1
        done
        sleep 30
    fi

    info "Stopping Docker containers..."
    if command -v docker >/dev/null 2>&1; then
        docker ps -q | xargs -r docker stop >/dev/null 2>&1
    fi

    info "Stopping User Scripts..."
    pkill -f "/tmp/user.scripts" 2>/dev/null || true

    info "Syncing disks..."
    sync

    echo ""
    echo "$ICON_REBOOT_SMART Rebooting system NOW..."
    sleep 5
    /sbin/reboot
}

# -----------------------------------------------------------------------------------------------
# CLEAN SHUTDOWN — trap SIGTERM/SIGINT from array stop
# -----------------------------------------------------------------------------------------------
WATCHDOG_RUNNING=true

cleanup() {
    echo ""
    info "System watchdog received shutdown signal — stopping cleanly"
    WATCHDOG_RUNNING=false
    exit 0
}

trap cleanup SIGTERM SIGINT

# -----------------------------------------------------------------------------------------------
# ━━━ CONTINUOUS MONITORING LOOP ━━━
# -----------------------------------------------------------------------------------------------
info "System watchdog started — checking every ${SYSTEM_WATCHDOG_INTERVAL}s"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

CYCLE=0

while [[ "$WATCHDOG_RUNNING" == true ]]; do
    ((CYCLE++))

    # Re-source Master.conf each cycle — picks up config changes without restart
    source "$SCRIPT_DIR/../Master.conf"
    SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 ))
    TOTAL_CORES=$(nproc)

    # Per-cycle triggers — cleared each iteration
    TRIGGERS=()

    # ── rootfs usage ─────────────────────────────────────────────────────────────────────────
    if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then
        ROOTFS_USED=$(df / --output=pcent | tail -1 | tr -d ' %')
        TRIGGERED=false
        [[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true
        run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \
            TRIGGERS+=("rootfs=${ROOTFS_USED}%")
    fi

    # ── /var/log usage ────────────────────────────────────────────────────────────────────────
    if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then
        LOG_USED=$(df -P /var/log | awk 'NR==2 {print $5}' | tr -d '%')
        TRIGGERED=false
        [[ "$LOG_USED" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true
        run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \
            TRIGGERS+=("log=${LOG_USED}%")
    fi

    # ── Free RAM ─────────────────────────────────────────────────────────────────────────────
    if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then
        MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
        MEM_GB=$((MEM_KB / 1024 / 1024))
        TRIGGERED=false
        [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]] && TRIGGERED=true
        run_strike_check "ram" "$TRIGGERED" "RAM ${MEM_GB}GB free" && \
            TRIGGERS+=("low_ram=${MEM_GB}GB")
    fi

    # ── ZFS ARC ──────────────────────────────────────────────────────────────────────────────
    if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
        ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
        ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
        ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX ))
        TRIGGERED=false
        if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then
            sync; echo 3 > /proc/sys/vm/drop_caches; sleep 5
            ARC_AFTER=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
            ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX ))
            [[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]] && TRIGGERED=true
        fi
        run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned" && \
            TRIGGERS+=("arc_pinned=${ARC_PCT}%")
    fi

    # ── CPU temperature ──────────────────────────────────────────────────────────────────────
    if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then
        CPU_TEMP=""
        if command -v sensors >/dev/null 2>&1; then
            CPU_TEMP=$(sensors 2>/dev/null | grep -i "Package id 0\|Tctl\|CPU Temp" | \
                awk '{print $NF}' | tr -d '+°C' | head -1)
        fi
        if [[ -n "$CPU_TEMP" ]]; then
            CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP")
            TRIGGERED=false
            [[ "$CPU_TEMP_INT" -ge "$SYS_WATCHDOG_CPU_TEMP_MAX" ]] && TRIGGERED=true
            run_strike_check "cpu_temp" "$TRIGGERED" "CPU temp ${CPU_TEMP_INT}°C" && \
                TRIGGERS+=("cpu_temp=${CPU_TEMP_INT}C")
        fi
    fi

    # ── Load average ─────────────────────────────────────────────────────────────────────────
    if [[ "$SYS_WATCHDOG_CHECK_LOAD" == true ]]; then
        LOAD=$(awk '{print $1}' /proc/loadavg)
        LOAD_INT=$(printf "%.0f" "$LOAD")
        LOAD_THRESHOLD=$(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER ))
        TRIGGERED=false
        [[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true
        run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \
            TRIGGERS+=("load=${LOAD}")
    fi

    # ── Zombie processes ─────────────────────────────────────────────────────────────────────
    if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then
        ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null)
        ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}"
        ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"
        ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}"
        TRIGGERED=false
        [[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true
        run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \
            TRIGGERS+=("zombies=${ZOMBIE_COUNT}")
    fi

    # ── Docker daemon ────────────────────────────────────────────────────────────────────────
    if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then
        TRIGGERED=false
        ! timeout 10 docker ps >/dev/null 2>&1 && TRIGGERED=true
        run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon unresponsive" && \
            TRIGGERS+=("docker_daemon")
    fi

    # ── Required containers from skip list ───────────────────────────────────────────────────
    if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
        FAILED_CONTAINERS=()
        while IFS= read -r container; do
            [[ -z "$container" ]] && continue
            STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
            [[ "$STATUS" != "true" ]] && FAILED_CONTAINERS+=("$container")
        done < "$SYS_WATCHDOG_FAILED_FILE"

        if [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]]; then
            run_strike_check "failed_containers" "true" "required containers stopped" && \
                TRIGGERS+=("containers=${FAILED_CONTAINERS[*]}")
        fi
    fi

    # ── Evaluate triggers ────────────────────────────────────────────────────────────────────
    if [[ ${#TRIGGERS[@]} -gt 0 ]]; then
        echo ""
        echo "━━━ $ICON_REBOOT_SMART System Watchdog — Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
        for t in "${TRIGGERS[@]}"; do
            echo "  $ICON_REBOOT_SMART $t"
        done
        echo ""
        do_reboot "${TRIGGERS[@]}"
    else
        log "Cycle $CYCLE — system healthy ($(date '+%H:%M:%S'))"
    fi

    # Sleep until next cycle — interruptible by SIGTERM
    sleep "$SYSTEM_WATCHDOG_INTERVAL" &
    wait $!

done