924 lines
39 KiB
Bash
924 lines
39 KiB
Bash
#!/bin/bash
|
|
# -----------------------------------------------------------------------------------------------
|
|
# --------------------------------- System Watchdog --------------------------------------------
|
|
# -----------------------------------------------------------------------------------------------
|
|
# Last line of defense — reboots the system cleanly if it is about to become unstable.
|
|
# Runs continuously as a background process — started by array_start.sh at array start.
|
|
# Works alongside docker_watchdog.sh which handles container-level healing first.
|
|
#
|
|
# Checks (all toggleable in Master.conf):
|
|
# rootfs usage — high rootfs fills rapidly when array is down, crash imminent
|
|
# /var/log usage — log spam can fill rootfs, indicates something is broken
|
|
# free RAM — critically low RAM means OOM or swap imminent
|
|
# ZFS ARC pinned — ARC not releasing after reclaim means memory is stuck
|
|
# CPU temperature — sustained tjmax causes throttling or kernel panic
|
|
# load average — sustained high load means something is stuck or runaway
|
|
# zombie processes — large zombie count indicates serious process management failure
|
|
# Docker daemon — unresponsive daemon means containers cannot be managed
|
|
# Required containers — stopped containers that should be running (after watchdog skip list)
|
|
#
|
|
# Abort conditions (toggleable):
|
|
# ZFS pool unhealthy — reboot with bad pool risks data loss
|
|
# Parity running — aborting parity is better than crashing mid-check
|
|
# Mover running — aborting move is better than crashing mid-move
|
|
#
|
|
# Reboot loop protection:
|
|
# Tracks reboot timestamps in persistent log on /boot/
|
|
# Rolling window — old entries purge automatically
|
|
# If reboot count hits limit in window → shutdown instead of reboot
|
|
#
|
|
# Continuous loop:
|
|
# Checks run every SYSTEM_WATCHDOG_INTERVAL seconds (default 300 = 5min)
|
|
# Master.conf re-sourced each cycle — config changes picked up without restart
|
|
# Silent when healthy — only verbose when trigger or reboot
|
|
# Clean shutdown on SIGTERM/SIGINT — sent by array stop
|
|
#
|
|
# All configuration in Master.conf under System Watchdog section.
|
|
# Supports --dry-run to show triggered conditions without rebooting.
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../Master.conf"
|
|
source "$SCRIPT_DIR/../common.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_GEAR Setup — runs once at start ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
success "Running as root"
|
|
|
|
acquire_lock
|
|
|
|
TOTAL_CORES=$(nproc)
|
|
|
|
# Ensure state and persistent files exist
|
|
touch "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || {
|
|
error "Cannot create state file: $SYS_WATCHDOG_STATE_FILE"
|
|
exit 1
|
|
}
|
|
|
|
touch "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || {
|
|
error "Cannot create reboot log: $SYS_WATCHDOG_REBOOT_LOG"
|
|
exit 1
|
|
}
|
|
|
|
touch "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || {
|
|
error "Cannot create failed container list: $SYS_WATCHDOG_FAILED_FILE"
|
|
exit 1
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SUMMARY Status ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HEALTH rootfs threshold: ${SYS_WATCHDOG_ROOTFS_PCT}% toggle: $SYS_WATCHDOG_CHECK_ROOTFS"
|
|
echo "$ICON_HEALTH log threshold: ${SYS_WATCHDOG_LOG_PCT}% toggle: $SYS_WATCHDOG_CHECK_LOG"
|
|
echo "$ICON_MEM RAM threshold: ${SYS_WATCHDOG_MEM_GB}GB free toggle: $SYS_WATCHDOG_CHECK_RAM"
|
|
echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}% toggle: $SYS_WATCHDOG_CHECK_ARC"
|
|
echo "$ICON_ZFS ARC release: ${SYS_WATCHDOG_ARC_RELEASE_PCT}%"
|
|
echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C toggle: $SYS_WATCHDOG_CHECK_CPU_TEMP"
|
|
echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x cores toggle: $SYS_WATCHDOG_CHECK_LOAD"
|
|
echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT} toggle: $SYS_WATCHDOG_CHECK_ZOMBIES"
|
|
echo "$ICON_CONTAINERS Docker daemon: toggle: $SYS_WATCHDOG_CHECK_DOCKER_DAEMON"
|
|
echo "$ICON_CONTAINERS Containers: toggle: $SYS_WATCHDOG_CHECK_CONTAINERS"
|
|
echo "$ICON_SHIELD Strike limit: $SYS_WATCHDOG_STRIKE_LIMIT"
|
|
echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s"
|
|
echo "$ICON_REBOOT_SMART Reboot limit: $SYS_WATCHDOG_REBOOT_LIMIT in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs"
|
|
echo "$ICON_ZFS Abort ZFS unhealthy: $SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY"
|
|
echo "$ICON_GEAR Abort on parity: $SYS_WATCHDOG_ABORT_ON_PARITY"
|
|
echo "$ICON_MOVER Abort on mover: $SYS_WATCHDOG_ABORT_ON_MOVER"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboot will be executed"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# STATE HELPERS — defined once, used every cycle
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
get_strikes() {
|
|
local key="$1"
|
|
grep -E "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2
|
|
}
|
|
|
|
set_strikes() {
|
|
local key="$1" count="$2"
|
|
grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
|
echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
|
mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE"
|
|
}
|
|
|
|
increment_strikes() {
|
|
local key="$1"
|
|
local current
|
|
current=$(get_strikes "$key")
|
|
[[ -z "$current" ]] && current=0
|
|
((current++))
|
|
set_strikes "$key" "$current"
|
|
echo "$current"
|
|
}
|
|
|
|
reset_strikes() {
|
|
local key="$1"
|
|
set_strikes "$key" 0
|
|
}
|
|
|
|
purge_old_reboots() {
|
|
local now
|
|
now=$(date +%s)
|
|
local cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW ))
|
|
grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do
|
|
[[ "$ts" -gt "$cutoff" ]] && echo "$ts"
|
|
done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp"
|
|
mv "${SYS_WATCHDOG_REBOOT_LOG}.tmp" "$SYS_WATCHDOG_REBOOT_LOG"
|
|
}
|
|
|
|
count_recent_reboots() {
|
|
purge_old_reboots
|
|
grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0
|
|
}
|
|
|
|
log_reboot() {
|
|
date +%s >> "$SYS_WATCHDOG_REBOOT_LOG"
|
|
}
|
|
|
|
check_abort_conditions() {
|
|
local should_abort=false
|
|
|
|
if command -v zpool >/dev/null 2>&1; then
|
|
local unhealthy
|
|
unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true)
|
|
if [[ -n "$unhealthy" ]]; then
|
|
if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then
|
|
error "$ICON_ZFS ZFS pool unhealthy — aborting reboot"
|
|
notify "System watchdog aborted reboot on $(hostname) — ZFS pool unhealthy" "System Watchdog" "warning"
|
|
should_abort=true
|
|
else
|
|
warn "$ICON_ZFS ZFS pool unhealthy — continuing reboot"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
if [[ -f /var/local/emhttp/parity-date.txt ]]; then
|
|
if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then
|
|
if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then
|
|
error "$ICON_DISK Parity check running — aborting reboot"
|
|
notify "System watchdog aborted reboot on $(hostname) — parity running" "System Watchdog" "warning"
|
|
should_abort=true
|
|
else
|
|
warn "$ICON_DISK Parity check running — continuing reboot"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
if pgrep -f "mover" >/dev/null 2>&1; then
|
|
if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then
|
|
error "$ICON_MOVER Mover running — aborting reboot"
|
|
notify "System watchdog aborted reboot on $(hostname) — mover running" "System Watchdog" "warning"
|
|
should_abort=true
|
|
else
|
|
warn "$ICON_MOVER Mover running — continuing reboot"
|
|
fi
|
|
fi
|
|
|
|
[[ "$should_abort" == true ]] && return 1
|
|
return 0
|
|
}
|
|
|
|
run_strike_check() {
|
|
local key="$1" triggered="$2" description="$3"
|
|
if [[ "$triggered" == true ]]; then
|
|
local strikes
|
|
strikes=$(increment_strikes "$key")
|
|
warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT"
|
|
if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then
|
|
error "$description hit strike limit — reboot triggered"
|
|
reset_strikes "$key"
|
|
return 0
|
|
fi
|
|
else
|
|
local current
|
|
current=$(get_strikes "$key")
|
|
if [[ -n "$current" && "$current" -gt 0 ]]; then
|
|
reset_strikes "$key"
|
|
fi
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
do_reboot() {
|
|
local triggers=("$@")
|
|
|
|
if ! check_abort_conditions; then
|
|
return
|
|
fi
|
|
|
|
RECENT_REBOOTS=$(count_recent_reboots)
|
|
info "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT"
|
|
|
|
if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then
|
|
error "Reboot limit hit — shutting down instead"
|
|
notify "Reboot loop detected on $(hostname) — shutting down after $RECENT_REBOOTS reboots — conditions: ${triggers[*]}" "System Watchdog" "warning"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would shutdown now"
|
|
return
|
|
fi
|
|
sync
|
|
/sbin/poweroff
|
|
return
|
|
fi
|
|
|
|
notify "System watchdog reboot triggered on $(hostname) — conditions: ${triggers[*]}" "System Watchdog" "warning"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — reboot sequence would begin now"
|
|
return
|
|
fi
|
|
|
|
log_reboot
|
|
|
|
info "Shutting down VMs..."
|
|
if command -v virsh >/dev/null 2>&1; then
|
|
for VM in $(virsh list --name 2>/dev/null); do
|
|
[[ -z "$VM" ]] && continue
|
|
virsh shutdown "$VM" >/dev/null 2>&1
|
|
done
|
|
sleep 30
|
|
fi
|
|
|
|
info "Stopping Docker containers..."
|
|
if command -v docker >/dev/null 2>&1; then
|
|
docker ps -q | xargs -r docker stop >/dev/null 2>&1
|
|
fi
|
|
|
|
info "Stopping User Scripts..."
|
|
pkill -f "/tmp/user.scripts" 2>/dev/null || true
|
|
|
|
info "Syncing disks..."
|
|
sync
|
|
|
|
echo ""
|
|
echo "$ICON_REBOOT_SMART Rebooting system NOW..."
|
|
sleep 5
|
|
/sbin/reboot
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# CLEAN SHUTDOWN — trap SIGTERM/SIGINT from array stop
|
|
# -----------------------------------------------------------------------------------------------
|
|
WATCHDOG_RUNNING=true
|
|
|
|
cleanup() {
|
|
echo ""
|
|
info "System watchdog received shutdown signal — stopping cleanly"
|
|
WATCHDOG_RUNNING=false
|
|
exit 0
|
|
}
|
|
|
|
trap cleanup SIGTERM SIGINT
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ CONTINUOUS MONITORING LOOP ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
info "System watchdog started — checking every ${SYSTEM_WATCHDOG_INTERVAL}s"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
CYCLE=0
|
|
|
|
while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
|
((CYCLE++))
|
|
|
|
# Re-source Master.conf each cycle — picks up config changes without restart
|
|
source "$SCRIPT_DIR/../Master.conf"
|
|
SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 ))
|
|
TOTAL_CORES=$(nproc)
|
|
|
|
# Per-cycle triggers — cleared each iteration
|
|
TRIGGERS=()
|
|
|
|
# ── rootfs usage ─────────────────────────────────────────────────────────────────────────
|
|
if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then
|
|
ROOTFS_USED=$(df / --output=pcent | tail -1 | tr -d ' %')
|
|
TRIGGERED=false
|
|
[[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true
|
|
run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \
|
|
TRIGGERS+=("rootfs=${ROOTFS_USED}%")
|
|
fi
|
|
|
|
# ── /var/log usage ────────────────────────────────────────────────────────────────────────
|
|
if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then
|
|
LOG_USED=$(df -P /var/log | awk 'NR==2 {print $5}' | tr -d '%')
|
|
TRIGGERED=false
|
|
[[ "$LOG_USED" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true
|
|
run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \
|
|
TRIGGERS+=("log=${LOG_USED}%")
|
|
fi
|
|
|
|
# ── Free RAM ─────────────────────────────────────────────────────────────────────────────
|
|
if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then
|
|
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
|
MEM_GB=$((MEM_KB / 1024 / 1024))
|
|
TRIGGERED=false
|
|
[[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]] && TRIGGERED=true
|
|
run_strike_check "ram" "$TRIGGERED" "RAM ${MEM_GB}GB free" && \
|
|
TRIGGERS+=("low_ram=${MEM_GB}GB")
|
|
fi
|
|
|
|
# ── ZFS ARC ──────────────────────────────────────────────────────────────────────────────
|
|
if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
|
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
|
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
|
ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX ))
|
|
TRIGGERED=false
|
|
if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then
|
|
sync; echo 3 > /proc/sys/vm/drop_caches; sleep 5
|
|
ARC_AFTER=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
|
ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX ))
|
|
[[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]] && TRIGGERED=true
|
|
fi
|
|
run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned" && \
|
|
TRIGGERS+=("arc_pinned=${ARC_PCT}%")
|
|
fi
|
|
|
|
# ── CPU temperature ──────────────────────────────────────────────────────────────────────
|
|
if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then
|
|
CPU_TEMP=""
|
|
if command -v sensors >/dev/null 2>&1; then
|
|
CPU_TEMP=$(sensors 2>/dev/null | grep -i "Package id 0\|Tctl\|CPU Temp" | \
|
|
awk '{print $NF}' | tr -d '+°C' | head -1)
|
|
fi
|
|
if [[ -n "$CPU_TEMP" ]]; then
|
|
CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP")
|
|
TRIGGERED=false
|
|
[[ "$CPU_TEMP_INT" -ge "$SYS_WATCHDOG_CPU_TEMP_MAX" ]] && TRIGGERED=true
|
|
run_strike_check "cpu_temp" "$TRIGGERED" "CPU temp ${CPU_TEMP_INT}°C" && \
|
|
TRIGGERS+=("cpu_temp=${CPU_TEMP_INT}C")
|
|
fi
|
|
fi
|
|
|
|
# ── Load average ─────────────────────────────────────────────────────────────────────────
|
|
if [[ "$SYS_WATCHDOG_CHECK_LOAD" == true ]]; then
|
|
LOAD=$(awk '{print $1}' /proc/loadavg)
|
|
LOAD_INT=$(printf "%.0f" "$LOAD")
|
|
LOAD_THRESHOLD=$(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER ))
|
|
TRIGGERED=false
|
|
[[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true
|
|
run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \
|
|
TRIGGERS+=("load=${LOAD}")
|
|
fi
|
|
|
|
# ── Zombie processes ─────────────────────────────────────────────────────────────────────
|
|
if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then
|
|
ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" || echo 0)
|
|
TRIGGERED=false
|
|
[[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true
|
|
run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \
|
|
TRIGGERS+=("zombies=${ZOMBIE_COUNT}")
|
|
fi
|
|
|
|
# ── Docker daemon ────────────────────────────────────────────────────────────────────────
|
|
if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then
|
|
TRIGGERED=false
|
|
! timeout 10 docker ps >/dev/null 2>&1 && TRIGGERED=true
|
|
run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon unresponsive" && \
|
|
TRIGGERS+=("docker_daemon")
|
|
fi
|
|
|
|
# ── Required containers from skip list ───────────────────────────────────────────────────
|
|
if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
|
FAILED_CONTAINERS=()
|
|
while IFS= read -r container; do
|
|
[[ -z "$container" ]] && continue
|
|
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
|
[[ "$STATUS" != "true" ]] && FAILED_CONTAINERS+=("$container")
|
|
done < "$SYS_WATCHDOG_FAILED_FILE"
|
|
|
|
if [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]]; then
|
|
run_strike_check "failed_containers" "true" "required containers stopped" && \
|
|
TRIGGERS+=("containers=${FAILED_CONTAINERS[*]}")
|
|
fi
|
|
fi
|
|
|
|
# ── Evaluate triggers ────────────────────────────────────────────────────────────────────
|
|
if [[ ${#TRIGGERS[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_REBOOT_SMART System Watchdog — Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
for t in "${TRIGGERS[@]}"; do
|
|
echo " $ICON_REBOOT_SMART $t"
|
|
done
|
|
echo ""
|
|
do_reboot "${TRIGGERS[@]}"
|
|
else
|
|
log "Cycle $CYCLE — system healthy ($(date '+%H:%M:%S'))"
|
|
fi
|
|
|
|
# Sleep until next cycle — interruptible by SIGTERM
|
|
sleep "$SYSTEM_WATCHDOG_INTERVAL" &
|
|
wait $!
|
|
|
|
done
|
|
#
|
|
# Checks (all toggleable in Master.conf):
|
|
# rootfs usage — high rootfs fills rapidly when array is down, crash imminent
|
|
# /var/log usage — log spam can fill rootfs, indicates something is broken
|
|
# free RAM — critically low RAM means OOM or swap imminent
|
|
# ZFS ARC pinned — ARC not releasing after reclaim means memory is stuck
|
|
# CPU temperature — sustained tjmax causes throttling or kernel panic
|
|
# load average — sustained high load means something is stuck or runaway
|
|
# zombie processes — large zombie count indicates serious process management failure
|
|
# Docker daemon — unresponsive daemon means containers cannot be managed
|
|
# Required containers — stopped containers that should be running (after watchdog skip list)
|
|
#
|
|
# Abort conditions (toggleable — true = abort, false = reboot anyway):
|
|
# ZFS pool unhealthy — reboot with bad pool risks data loss
|
|
# Parity running — aborting parity is better than crashing mid-check
|
|
# Mover running — aborting move is better than crashing mid-move
|
|
#
|
|
# Reboot loop protection:
|
|
# Tracks reboot timestamps in persistent log on /boot/
|
|
# Rolling window — old entries purge automatically after SYS_WATCHDOG_REBOOT_WINDOW_HRS
|
|
# If reboot count hits SYS_WATCHDOG_REBOOT_LIMIT in window → shutdown instead of reboot
|
|
#
|
|
# All configuration in Master.conf under System Watchdog section.
|
|
# Supports --dry-run to show triggered conditions without rebooting.
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../Master.conf"
|
|
source "$SCRIPT_DIR/../common.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# Convert window hours to seconds for internal use
|
|
SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 ))
|
|
|
|
TOTAL_CORES=$(nproc)
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_GEAR Setup ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
success "Running as root"
|
|
|
|
acquire_lock
|
|
|
|
# Ensure state and persistent files exist
|
|
touch "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || {
|
|
error "Cannot create state file: $SYS_WATCHDOG_STATE_FILE"
|
|
exit 1
|
|
}
|
|
|
|
touch "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || {
|
|
error "Cannot create reboot log: $SYS_WATCHDOG_REBOOT_LOG"
|
|
exit 1
|
|
}
|
|
|
|
touch "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || {
|
|
error "Cannot create failed container list: $SYS_WATCHDOG_FAILED_FILE"
|
|
exit 1
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SUMMARY Status ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HEALTH rootfs threshold: ${SYS_WATCHDOG_ROOTFS_PCT}% toggle: $SYS_WATCHDOG_CHECK_ROOTFS"
|
|
echo "$ICON_HEALTH log threshold: ${SYS_WATCHDOG_LOG_PCT}% toggle: $SYS_WATCHDOG_CHECK_LOG"
|
|
echo "$ICON_MEM RAM threshold: ${SYS_WATCHDOG_MEM_GB}GB free toggle: $SYS_WATCHDOG_CHECK_RAM"
|
|
echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}% toggle: $SYS_WATCHDOG_CHECK_ARC"
|
|
echo "$ICON_ZFS ARC release: ${SYS_WATCHDOG_ARC_RELEASE_PCT}%"
|
|
echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C toggle: $SYS_WATCHDOG_CHECK_CPU_TEMP"
|
|
echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x cores toggle: $SYS_WATCHDOG_CHECK_LOAD"
|
|
echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT} toggle: $SYS_WATCHDOG_CHECK_ZOMBIES"
|
|
echo "$ICON_CONTAINERS Docker daemon: toggle: $SYS_WATCHDOG_CHECK_DOCKER_DAEMON"
|
|
echo "$ICON_CONTAINERS Containers: toggle: $SYS_WATCHDOG_CHECK_CONTAINERS"
|
|
echo "$ICON_SHIELD Strike limit: $SYS_WATCHDOG_STRIKE_LIMIT"
|
|
echo "$ICON_REBOOT_SMART Reboot limit: $SYS_WATCHDOG_REBOOT_LIMIT in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs"
|
|
echo "$ICON_ZFS Abort ZFS unhealthy: $SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY"
|
|
echo "$ICON_GEAR Abort on parity: $SYS_WATCHDOG_ABORT_ON_PARITY"
|
|
echo "$ICON_MOVER Abort on mover: $SYS_WATCHDOG_ABORT_ON_MOVER"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboot will be executed"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# STATE HELPERS
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
get_strikes() {
|
|
local key="$1"
|
|
grep -E "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2
|
|
}
|
|
|
|
set_strikes() {
|
|
local key="$1" count="$2"
|
|
grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
|
echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
|
mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE"
|
|
}
|
|
|
|
# Returns current strike count, increments and saves, then echoes new count
|
|
increment_strikes() {
|
|
local key="$1"
|
|
local current
|
|
current=$(get_strikes "$key")
|
|
[[ -z "$current" ]] && current=0
|
|
((current++))
|
|
set_strikes "$key" "$current"
|
|
echo "$current"
|
|
}
|
|
|
|
reset_strikes() {
|
|
local key="$1"
|
|
set_strikes "$key" 0
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# REBOOT LOG HELPERS
|
|
# Tracks reboot timestamps for loop detection.
|
|
# Rolling window — entries older than SYS_WATCHDOG_REBOOT_WINDOW are purged automatically.
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
purge_old_reboots() {
|
|
local now
|
|
now=$(date +%s)
|
|
local cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW ))
|
|
|
|
grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do
|
|
[[ "$ts" -gt "$cutoff" ]] && echo "$ts"
|
|
done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp"
|
|
mv "${SYS_WATCHDOG_REBOOT_LOG}.tmp" "$SYS_WATCHDOG_REBOOT_LOG"
|
|
}
|
|
|
|
count_recent_reboots() {
|
|
purge_old_reboots
|
|
grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0
|
|
}
|
|
|
|
log_reboot() {
|
|
date +%s >> "$SYS_WATCHDOG_REBOOT_LOG"
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ABORT CONDITION CHECKS
|
|
# Run before any reboot is triggered — abort conditions prevent rebooting
|
|
# when it would make things worse than letting the system run.
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
check_abort_conditions() {
|
|
local should_abort=false
|
|
|
|
# ZFS pool health
|
|
if command -v zpool >/dev/null 2>&1; then
|
|
local unhealthy
|
|
unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true)
|
|
if [[ -n "$unhealthy" ]]; then
|
|
if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then
|
|
error "$ICON_ZFS ZFS pool unhealthy — aborting reboot (ABORT_ON_ZFS_UNHEALTHY=true)"
|
|
notify "System watchdog aborted reboot on $(hostname) — ZFS pool unhealthy" "System Watchdog" "warning"
|
|
should_abort=true
|
|
else
|
|
warn "$ICON_ZFS ZFS pool unhealthy — continuing reboot (ABORT_ON_ZFS_UNHEALTHY=false)"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Parity check running
|
|
if [[ -f /var/local/emhttp/parity-date.txt ]]; then
|
|
if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then
|
|
if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then
|
|
error "$ICON_DISK Parity check running — aborting reboot (ABORT_ON_PARITY=true)"
|
|
notify "System watchdog aborted reboot on $(hostname) — parity check running" "System Watchdog" "warning"
|
|
should_abort=true
|
|
else
|
|
warn "$ICON_DISK Parity check running — continuing reboot (ABORT_ON_PARITY=false)"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Mover running
|
|
if pgrep -f "mover" >/dev/null 2>&1; then
|
|
if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then
|
|
error "$ICON_MOVER Mover is running — aborting reboot (ABORT_ON_MOVER=true)"
|
|
notify "System watchdog aborted reboot on $(hostname) — mover running" "System Watchdog" "warning"
|
|
should_abort=true
|
|
else
|
|
warn "$ICON_MOVER Mover is running — continuing reboot (ABORT_ON_MOVER=false)"
|
|
fi
|
|
fi
|
|
|
|
[[ "$should_abort" == true ]] && return 1
|
|
return 0
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# STRIKE-BASED CHECK HELPER
|
|
# Runs a check function, increments strikes on trigger, resets on clear.
|
|
# Returns 0 if strike limit hit (reboot trigger), 1 otherwise.
|
|
# Usage: run_strike_check "key" "triggered (true/false)" "description"
|
|
# -----------------------------------------------------------------------------------------------
|
|
run_strike_check() {
|
|
local key="$1" triggered="$2" description="$3"
|
|
|
|
if [[ "$triggered" == true ]]; then
|
|
local strikes
|
|
strikes=$(increment_strikes "$key")
|
|
warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT"
|
|
if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then
|
|
error "$description hit strike limit — reboot triggered"
|
|
reset_strikes "$key"
|
|
return 0
|
|
fi
|
|
else
|
|
local current
|
|
current=$(get_strikes "$key")
|
|
if [[ -n "$current" && "$current" -gt 0 ]]; then
|
|
info "$description — recovered, resetting strikes"
|
|
reset_strikes "$key"
|
|
fi
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SHIELD Health Checks ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Health Checks — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo ""
|
|
|
|
TRIGGERS=()
|
|
|
|
# rootfs usage
|
|
if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then
|
|
info "$ICON_HEALTH Checking rootfs usage..."
|
|
ROOTFS_USED=$(df / --output=pcent | tail -1 | tr -d ' %')
|
|
TRIGGERED=false
|
|
if [[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]]; then
|
|
error "$ICON_HEALTH rootfs is ${ROOTFS_USED}% full — threshold ${SYS_WATCHDOG_ROOTFS_PCT}%"
|
|
TRIGGERED=true
|
|
else
|
|
success "$ICON_HEALTH rootfs: ${ROOTFS_USED}% used"
|
|
fi
|
|
run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && TRIGGERS+=("rootfs=${ROOTFS_USED}%")
|
|
fi
|
|
|
|
# /var/log usage
|
|
if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then
|
|
info "$ICON_HEALTH Checking /var/log usage..."
|
|
LOG_USED=$(df -P /var/log | awk 'NR==2 {print $5}' | tr -d '%')
|
|
TRIGGERED=false
|
|
if [[ "$LOG_USED" -ge "$SYS_WATCHDOG_LOG_PCT" ]]; then
|
|
error "$ICON_HEALTH /var/log is ${LOG_USED}% full — threshold ${SYS_WATCHDOG_LOG_PCT}%"
|
|
TRIGGERED=true
|
|
else
|
|
success "$ICON_HEALTH /var/log: ${LOG_USED}% used"
|
|
fi
|
|
run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && TRIGGERS+=("log=${LOG_USED}%")
|
|
fi
|
|
|
|
# Free RAM
|
|
if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then
|
|
info "$ICON_MEM Checking available RAM..."
|
|
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
|
MEM_GB=$((MEM_KB / 1024 / 1024))
|
|
TRIGGERED=false
|
|
if [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]]; then
|
|
error "$ICON_MEM Available RAM: ${MEM_GB}GB — threshold ${SYS_WATCHDOG_MEM_GB}GB"
|
|
TRIGGERED=true
|
|
else
|
|
success "$ICON_MEM Available RAM: ${MEM_GB}GB"
|
|
fi
|
|
run_strike_check "ram" "$TRIGGERED" "RAM ${MEM_GB}GB free" && TRIGGERS+=("low_ram=${MEM_GB}GB")
|
|
fi
|
|
|
|
# ZFS ARC pinned
|
|
if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
|
info "$ICON_ZFS Checking ZFS ARC..."
|
|
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
|
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
|
ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX ))
|
|
TRIGGERED=false
|
|
|
|
if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then
|
|
warn "$ICON_ZFS ARC at ${ARC_PCT}% — attempting reclaim..."
|
|
sync
|
|
echo 3 > /proc/sys/vm/drop_caches
|
|
sleep 5
|
|
ARC_AFTER=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
|
ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX ))
|
|
if [[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]]; then
|
|
error "$ICON_ZFS ARC still ${ARC_AFTER_PCT}% after reclaim — threshold ${SYS_WATCHDOG_ARC_RELEASE_PCT}%"
|
|
TRIGGERED=true
|
|
else
|
|
success "$ICON_ZFS ARC released to ${ARC_AFTER_PCT}% after reclaim"
|
|
fi
|
|
else
|
|
success "$ICON_ZFS ARC: ${ARC_PCT}% of max"
|
|
fi
|
|
run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned" && TRIGGERS+=("arc_pinned=${ARC_PCT}%")
|
|
elif [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]]; then
|
|
info "$ICON_ZFS ZFS arcstats not available — skipping"
|
|
fi
|
|
|
|
# CPU temperature
|
|
if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then
|
|
info "$ICON_GEAR Checking CPU temperature..."
|
|
CPU_TEMP=""
|
|
if command -v sensors >/dev/null 2>&1; then
|
|
CPU_TEMP=$(sensors 2>/dev/null | grep -i "Package id 0\|Tctl\|CPU Temp" | awk '{print $NF}' | tr -d '+°C' | head -1)
|
|
fi
|
|
|
|
if [[ -z "$CPU_TEMP" ]]; then
|
|
info "$ICON_GEAR CPU temperature sensor not available — skipping"
|
|
else
|
|
CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP")
|
|
TRIGGERED=false
|
|
if [[ "$CPU_TEMP_INT" -ge "$SYS_WATCHDOG_CPU_TEMP_MAX" ]]; then
|
|
error "$ICON_GEAR CPU temp ${CPU_TEMP_INT}°C — threshold ${SYS_WATCHDOG_CPU_TEMP_MAX}°C"
|
|
TRIGGERED=true
|
|
else
|
|
success "$ICON_GEAR CPU temp: ${CPU_TEMP_INT}°C"
|
|
fi
|
|
run_strike_check "cpu_temp" "$TRIGGERED" "CPU temp ${CPU_TEMP_INT}°C" && TRIGGERS+=("cpu_temp=${CPU_TEMP_INT}C")
|
|
fi
|
|
fi
|
|
|
|
# Load average
|
|
if [[ "$SYS_WATCHDOG_CHECK_LOAD" == true ]]; then
|
|
info "$ICON_GEAR Checking load average..."
|
|
LOAD=$(awk '{print $1}' /proc/loadavg)
|
|
LOAD_INT=$(printf "%.0f" "$LOAD")
|
|
LOAD_THRESHOLD=$(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER ))
|
|
TRIGGERED=false
|
|
if [[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]]; then
|
|
error "$ICON_GEAR Load average ${LOAD} — threshold ${LOAD_THRESHOLD} (${TOTAL_CORES} cores x ${SYS_WATCHDOG_LOAD_MULTIPLIER})"
|
|
TRIGGERED=true
|
|
else
|
|
success "$ICON_GEAR Load average: ${LOAD}"
|
|
fi
|
|
run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && TRIGGERS+=("load=${LOAD}")
|
|
fi
|
|
|
|
# Zombie processes
|
|
if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then
|
|
info "$ICON_GEAR Checking zombie processes..."
|
|
ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" || echo 0)
|
|
TRIGGERED=false
|
|
if [[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]]; then
|
|
error "$ICON_GEAR Zombie processes: $ZOMBIE_COUNT — threshold $SYS_WATCHDOG_ZOMBIE_LIMIT"
|
|
TRIGGERED=true
|
|
else
|
|
success "$ICON_GEAR Zombie processes: $ZOMBIE_COUNT"
|
|
fi
|
|
run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && TRIGGERS+=("zombies=${ZOMBIE_COUNT}")
|
|
fi
|
|
|
|
# Docker daemon health
|
|
if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then
|
|
info "$ICON_CONTAINERS Checking Docker daemon..."
|
|
TRIGGERED=false
|
|
if ! timeout 10 docker ps >/dev/null 2>&1; then
|
|
error "$ICON_CONTAINERS Docker daemon is not responding"
|
|
TRIGGERED=true
|
|
else
|
|
success "$ICON_CONTAINERS Docker daemon is healthy"
|
|
fi
|
|
run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon unresponsive" && TRIGGERS+=("docker_daemon")
|
|
fi
|
|
|
|
# Required containers from skip list
|
|
if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
|
info "$ICON_CONTAINERS Checking persistent failed containers..."
|
|
FAILED_CONTAINERS=()
|
|
while IFS= read -r container; do
|
|
[[ -z "$container" ]] && continue
|
|
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
|
if [[ "$STATUS" != "true" ]]; then
|
|
error "$ICON_NOT_RUNNING $container still not running (from skip list)"
|
|
FAILED_CONTAINERS+=("$container")
|
|
fi
|
|
done < "$SYS_WATCHDOG_FAILED_FILE"
|
|
|
|
if [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]]; then
|
|
TRIGGERED=true
|
|
run_strike_check "failed_containers" "$TRIGGERED" "required containers stopped" && TRIGGERS+=("containers=${FAILED_CONTAINERS[*]}")
|
|
fi
|
|
fi
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# No triggers — exit cleanly
|
|
# -----------------------------------------------------------------------------------------------
|
|
if [[ ${#TRIGGERS[@]} -eq 0 ]]; then
|
|
echo ""
|
|
success "No reboot conditions met — system is healthy"
|
|
exit 0
|
|
fi
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_REBOOT_SMART Reboot Triggered ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_REBOOT_SMART Reboot Triggered ━━━"
|
|
for t in "${TRIGGERS[@]}"; do
|
|
echo " $ICON_REBOOT_SMART $t"
|
|
done
|
|
echo ""
|
|
|
|
# Check abort conditions before proceeding
|
|
if ! check_abort_conditions; then
|
|
exit 0
|
|
fi
|
|
|
|
# Reboot loop protection — check recent reboot count
|
|
RECENT_REBOOTS=$(count_recent_reboots)
|
|
info "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT"
|
|
|
|
if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then
|
|
error "Reboot limit hit — $RECENT_REBOOTS reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs — shutting down instead"
|
|
notify "Reboot loop detected on $(hostname) — shutting down after $RECENT_REBOOTS reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs — conditions: ${TRIGGERS[*]}" "System Watchdog" "warning"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would shutdown now"
|
|
exit 0
|
|
fi
|
|
|
|
sync
|
|
/sbin/poweroff
|
|
exit 0
|
|
fi
|
|
|
|
notify "System watchdog reboot triggered on $(hostname) — conditions: ${TRIGGERS[*]}" "System Watchdog" "warning"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — reboot sequence would begin now"
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY SYSTEM WATCHDOG SUMMARY ━━━━━"
|
|
echo "$ICON_REBOOT_SMART Triggers: ${TRIGGERS[*]}"
|
|
echo "$ICON_REBOOT_SMART Recent reboots: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT"
|
|
echo "$ICON_WARN Status: DRY RUN — no reboot executed"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# Graceful shutdown sequence
|
|
# -----------------------------------------------------------------------------------------------
|
|
info "Logging reboot timestamp..."
|
|
log_reboot
|
|
|
|
info "Shutting down VMs..."
|
|
if command -v virsh >/dev/null 2>&1; then
|
|
for VM in $(virsh list --name 2>/dev/null); do
|
|
[[ -z "$VM" ]] && continue
|
|
info "Shutting down VM: $VM"
|
|
virsh shutdown "$VM" >/dev/null 2>&1
|
|
done
|
|
info "Waiting 30s for VMs..."
|
|
sleep 30
|
|
fi
|
|
|
|
info "Stopping Docker containers..."
|
|
if command -v docker >/dev/null 2>&1; then
|
|
docker ps -q | xargs -r docker stop >/dev/null 2>&1
|
|
success "Docker containers stopped"
|
|
fi
|
|
|
|
info "Stopping User Scripts..."
|
|
pkill -f "/tmp/user.scripts" 2>/dev/null || true
|
|
|
|
info "Syncing disks..."
|
|
sync
|
|
success "Disks synced"
|
|
|
|
echo ""
|
|
echo "$ICON_REBOOT_SMART Rebooting system NOW..."
|
|
sleep 5
|
|
/sbin/reboot |