From 6df0bafc090fef30dc9a54031425c671d7da1dc1 Mon Sep 17 00:00:00 2001 From: FailedProxy Date: Fri, 24 Apr 2026 16:26:35 -0400 Subject: [PATCH] fixed lock logic. if a continuious app hits a lock it skips to the next instead of error --- Docker_Essentials/docker_watchdog.sh | 1428 ++++++-------------------- Master.conf | 37 +- common.sh | 21 +- unRAID_Essentials/system_watchdog.sh | 495 +-------- 4 files changed, 347 insertions(+), 1634 deletions(-) diff --git a/Docker_Essentials/docker_watchdog.sh b/Docker_Essentials/docker_watchdog.sh index 3663fa6..a423d33 100644 --- a/Docker_Essentials/docker_watchdog.sh +++ b/Docker_Essentials/docker_watchdog.sh @@ -1,44 +1,40 @@ #!/bin/bash # ----------------------------------------------------------------------------------------------- -# --------------------------------- Docker Watchdog -------------------------------------------- +# --------------------------------- System Watchdog -------------------------------------------- # ----------------------------------------------------------------------------------------------- -# Two-tier self-healing container monitoring system — runs continuously as a background process. -# Started by array_start.sh at array start — runs until array stops or SIGTERM received. +# Last line of defense — reboots the system cleanly if it is about to become unstable. +# Runs continuously as a background process — started by array_start.sh at array start. +# Works alongside docker_watchdog.sh which handles container-level healing first. # -# Tier 1 — Strict monitoring (configured containers only) -# Memory hard limits — immediate restart if exceeded -# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT strikes -# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT strikes -# Required containers — must always be running, strike system with skip list +# Checks (all toggleable in Master.conf): +# rootfs usage — high rootfs fills rapidly when array is down, crash imminent +# /var/log usage — log spam can fill rootfs, indicates something is broken +# free RAM — critically low RAM means OOM or swap imminent +# ZFS ARC pinned — ARC not releasing after reclaim means memory is stuck +# CPU temperature — sustained tjmax causes throttling or kernel panic +# load average — sustained high load means something is stuck or runaway +# zombie processes — large zombie count indicates serious process management failure +# Docker daemon — unresponsive daemon means containers cannot be managed +# Required containers — stopped containers that should be running (after watchdog skip list) # -# Tier 2 — Global health scan (all running containers) -# Unhealthy status — Docker HEALTHCHECK unhealthy → restart -# OOM killed — kernel killed container → restart + notify -# Crash loop detection — RestartCount climbing → notify, critical above limit -# Dead containers — remove and restart -# Unexpected exits — non-zero exit code → restart +# Abort conditions (toggleable): +# ZFS pool unhealthy — reboot with bad pool risks data loss +# Parity running — aborting parity is better than crashing mid-check +# Mover running — aborting move is better than crashing mid-move # -# Cross-cutting intelligence: -# Startup grace period — skip restarts while system is still booting -# Dependency ordering — restart database before app -# Restart loop protect — stop restarting after X restarts in X hours → skip list -# Skip list auto-clear — clears when container recovers -# Notification batching — one clean summary per cycle, not one ping per event -# Quiet when healthy — only logs when something needs attention -# Parity awareness — skips restarts during parity check +# Reboot loop protection: +# Tracks reboot timestamps in persistent log on /boot/ +# Rolling window — old entries purge automatically +# If reboot count hits limit in window → shutdown instead of reboot # # Continuous loop: -# Checks run every DOCKER_WATCHDOG_INTERVAL seconds (default 900 = 15min) +# Checks run every SYSTEM_WATCHDOG_INTERVAL seconds (default 300 = 5min) +# Master.conf re-sourced each cycle — config changes picked up without restart +# Silent when healthy — only verbose when trigger or reboot # Clean shutdown on SIGTERM/SIGINT — sent by array stop -# Variables scoped per-cycle — no state accumulation between cycles # -# State files: -# WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot) -# SYS_WATCHDOG_FAILED_FILE — persistent skip list (/boot — survives reboots) -# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection -# -# All configuration in Master.conf under Docker Watchdog section. -# Supports --dry-run and --status. +# All configuration in Master.conf under System Watchdog section. +# Supports --dry-run to show triggered conditions without rebooting. # ----------------------------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -61,39 +57,25 @@ fi success "Running as root" -acquire_lock +acquire_lock "continuous" -# Select correct per-host watchdog lists — done once at startup -detect_hosts +TOTAL_CORES=$(nproc) -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}") - declare -A WATCHDOG_CONTAINER_URLS - for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do - WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}" - done -else - WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}") - declare -A WATCHDOG_CONTAINER_URLS - for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do - WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}" - done -fi - -info "Watchdog running as: $LOCAL_SERVER_NAME" -info "Check interval: ${DOCKER_WATCHDOG_INTERVAL}s" - -if ! command -v docker >/dev/null 2>&1; then - error "Docker not found" +# Ensure state and persistent files exist +touch "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || { + error "Cannot create state file: $SYS_WATCHDOG_STATE_FILE" exit 1 -fi +} -success "Docker found" -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" +touch "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || { + error "Cannot create reboot log: $SYS_WATCHDOG_REBOOT_LOG" + exit 1 +} -# Ensure state files exist -touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \ - "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null +touch "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || { + error "Cannot create failed container list: $SYS_WATCHDOG_FAILED_FILE" + exit 1 +} # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Status ━━━ @@ -101,158 +83,198 @@ touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \ if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_CONTAINERS Watched: ${!WATCHDOG_CONTAINERS[@]}" - echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}" - echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL" - echo "$ICON_WATCHDOG Interval: ${DOCKER_WATCHDOG_INTERVAL}s" - echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s" - echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" - echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY" - echo "$ICON_WATCHDOG Ignore list: ${WATCHDOG_SCAN_IGNORE[*]:-none}" - echo "$ICON_GEAR Dry Run: $DRY_RUN" - UPTIME_S=$(awk '{print int($1)}' /proc/uptime) - echo "$ICON_TIME System uptime: $(format_duration $UPTIME_S)" - [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]] && \ - warn "Within startup grace period — restarts suppressed" + echo "$ICON_HEALTH rootfs threshold: ${SYS_WATCHDOG_ROOTFS_PCT}% toggle: $SYS_WATCHDOG_CHECK_ROOTFS" + echo "$ICON_HEALTH log threshold: ${SYS_WATCHDOG_LOG_PCT}% toggle: $SYS_WATCHDOG_CHECK_LOG" + echo "$ICON_MEM RAM threshold: ${SYS_WATCHDOG_MEM_GB}GB free toggle: $SYS_WATCHDOG_CHECK_RAM" + echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}% toggle: $SYS_WATCHDOG_CHECK_ARC" + echo "$ICON_ZFS ARC release: ${SYS_WATCHDOG_ARC_RELEASE_PCT}%" + echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C toggle: $SYS_WATCHDOG_CHECK_CPU_TEMP" + echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x cores toggle: $SYS_WATCHDOG_CHECK_LOAD" + echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT} toggle: $SYS_WATCHDOG_CHECK_ZOMBIES" + echo "$ICON_CONTAINERS Docker daemon: toggle: $SYS_WATCHDOG_CHECK_DOCKER_DAEMON" + echo "$ICON_CONTAINERS Containers: toggle: $SYS_WATCHDOG_CHECK_CONTAINERS" + echo "$ICON_SHIELD Strike limit: $SYS_WATCHDOG_STRIKE_LIMIT" + echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s" + echo "$ICON_REBOOT_SMART Reboot limit: $SYS_WATCHDOG_REBOOT_LIMIT in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs" + echo "$ICON_ZFS Abort ZFS unhealthy: $SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" + echo "$ICON_GEAR Abort on parity: $SYS_WATCHDOG_ABORT_ON_PARITY" + echo "$ICON_MOVER Abort on mover: $SYS_WATCHDOG_ABORT_ON_MOVER" + echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboot will be executed" + # ----------------------------------------------------------------------------------------------- -# HELPERS — defined once, used every cycle +# STATE HELPERS — defined once, used every cycle # ----------------------------------------------------------------------------------------------- get_strikes() { - grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0" + local key="$1" + grep -E "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2 } set_strikes() { - local container="$1" count="$2" file="$3" - if grep -q "^${container}:" "$file" 2>/dev/null; then - sed -i "s/^${container}:.*/${container}:${count}/" "$file" - else - echo "${container}:${count}" >> "$file" - fi + local key="$1" count="$2" + grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp" + echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp" + mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE" } -is_skipped() { - grep -q "^${1}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null +increment_strikes() { + local key="$1" + local current + current=$(get_strikes "$key") + [[ -z "$current" ]] && current=0 + ((current++)) + set_strikes "$key" "$current" + echo "$current" } -add_to_skip_list() { - local container="$1" reason="$2" - if ! is_skipped "$container"; then - echo "$container" >> "$SYS_WATCHDOG_FAILED_FILE" - error "$container added to skip list — $reason" - queue_notify "$container added to skip list on $(hostname) — $reason — manual intervention needed" "critical" - fi +reset_strikes() { + local key="$1" + set_strikes "$key" 0 } -remove_from_skip_list() { - sed -i "/^${1}$/d" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null - success "$1 removed from skip list — recovered" -} - -log_restart() { - local container="$1" +purge_old_reboots() { local now - now=$(date '+%Y-%m-%d %H:%M:%S') - local cutoff - cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S') - echo "${container}|${now}" >> "$WATCHDOG_CONTAINER_RESTART_LOG" - local tmp="${WATCHDOG_CONTAINER_RESTART_LOG}.tmp" - awk -F'|' -v cutoff="$cutoff" '$2 >= cutoff' \ - "$WATCHDOG_CONTAINER_RESTART_LOG" > "$tmp" && \ - mv "$tmp" "$WATCHDOG_CONTAINER_RESTART_LOG" + now=$(date +%s) + local cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW )) + grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do + [[ "$ts" -gt "$cutoff" ]] && echo "$ts" + done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp" + mv "${SYS_WATCHDOG_REBOOT_LOG}.tmp" "$SYS_WATCHDOG_REBOOT_LOG" } -get_restart_count() { - local container="$1" - local cutoff - cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S') - awk -F'|' -v c="$container" -v cutoff="$cutoff" \ - '$1==c && $2>=cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l +count_recent_reboots() { + purge_old_reboots + grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0 } -dependencies_satisfied() { - local container="$1" - local deps="${WATCHDOG_DEPENDENCIES[$container]:-}" - [[ -z "$deps" ]] && return 0 - for dep in $deps; do - local status - status=$(docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null) - if [[ "$status" != "true" ]]; then - warn "$container — dependency $dep is not running — skipping restart this cycle" - return 1 +log_reboot() { + date +%s >> "$SYS_WATCHDOG_REBOOT_LOG" +} + +check_abort_conditions() { + local should_abort=false + + if command -v zpool >/dev/null 2>&1; then + local unhealthy + unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true) + if [[ -n "$unhealthy" ]]; then + if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then + error "$ICON_ZFS ZFS pool unhealthy — aborting reboot" + notify "System watchdog aborted reboot on $(hostname) — ZFS pool unhealthy" "System Watchdog" "warning" + should_abort=true + else + warn "$ICON_ZFS ZFS pool unhealthy — continuing reboot" + fi fi - done + fi + + if [[ -f /var/local/emhttp/parity-date.txt ]]; then + if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then + if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then + error "$ICON_DISK Parity check running — aborting reboot" + notify "System watchdog aborted reboot on $(hostname) — parity running" "System Watchdog" "warning" + should_abort=true + else + warn "$ICON_DISK Parity check running — continuing reboot" + fi + fi + fi + + if pgrep -f "mover" >/dev/null 2>&1; then + if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then + error "$ICON_MOVER Mover running — aborting reboot" + notify "System watchdog aborted reboot on $(hostname) — mover running" "System Watchdog" "warning" + should_abort=true + else + warn "$ICON_MOVER Mover running — continuing reboot" + fi + fi + + [[ "$should_abort" == true ]] && return 1 return 0 } -safe_restart() { - local container="$1" reason="$2" - local restart_count - restart_count=$(get_restart_count "$container") - if [[ "$restart_count" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then - add_to_skip_list "$container" \ - "restarted $restart_count times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" - return 2 +run_strike_check() { + local key="$1" triggered="$2" description="$3" + if [[ "$triggered" == true ]]; then + local strikes + strikes=$(increment_strikes "$key") + warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT" + if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then + error "$description hit strike limit — reboot triggered" + reset_strikes "$key" + return 0 + fi + else + local current + current=$(get_strikes "$key") + if [[ -n "$current" && "$current" -gt 0 ]]; then + reset_strikes "$key" + fi fi - dependencies_satisfied "$container" || return 1 - local uptime_s - uptime_s=$(awk '{print int($1)}' /proc/uptime) - if [[ "$uptime_s" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then - warn "$container — within startup grace period — skipping restart" - return 1 + return 1 +} + +do_reboot() { + local triggers=("$@") + + if ! check_abort_conditions; then + return fi + + RECENT_REBOOTS=$(count_recent_reboots) + info "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT" + + if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then + error "Reboot limit hit — shutting down instead" + notify "Reboot loop detected on $(hostname) — shutting down after $RECENT_REBOOTS reboots — conditions: ${triggers[*]}" "System Watchdog" "warning" + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would shutdown now" + return + fi + sync + /sbin/poweroff + return + fi + + notify "System watchdog reboot triggered on $(hostname) — conditions: ${triggers[*]}" "System Watchdog" "warning" + if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would restart $container ($reason)" - return 0 + warn "DRY RUN — reboot sequence would begin now" + return fi - info "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]..." - if docker restart "$container" >/dev/null 2>&1; then - success "$ICON_STARTED $container restarted" - log_restart "$container" - return 0 - else - error "Failed to restart $container" - return 1 - fi -} -queue_notify() { - local message="$1" severity="${2:-warning}" - NOTIFY_EVENTS+=("${severity}|${message}") - log "Queued: $message" -} + log_reboot -flush_notify() { - [[ ${#NOTIFY_EVENTS[@]} -eq 0 ]] && return - if [[ "$WATCHDOG_BATCH_NOTIFY" == "true" ]]; then - local highest_severity="normal" - local messages=() - for event in "${NOTIFY_EVENTS[@]}"; do - local sev="${event%%|*}" msg="${event#*|}" - messages+=("$msg") - [[ "$sev" == "critical" ]] && highest_severity="warning" - [[ "$sev" == "warning" && "$highest_severity" == "normal" ]] && highest_severity="warning" - done - local summary - summary=$(printf '%s. ' "${messages[@]}") - notify "Docker Watchdog on $(hostname) — ${#NOTIFY_EVENTS[@]} event(s): $summary" \ - "Docker Watchdog" "$highest_severity" - else - for event in "${NOTIFY_EVENTS[@]}"; do - local sev="${event%%|*}" msg="${event#*|}" - [[ "$sev" == "critical" ]] && sev="warning" - notify "$msg" "Docker Watchdog" "$sev" + info "Shutting down VMs..." + if command -v virsh >/dev/null 2>&1; then + for VM in $(virsh list --name 2>/dev/null); do + [[ -z "$VM" ]] && continue + virsh shutdown "$VM" >/dev/null 2>&1 done + sleep 30 fi - NOTIFY_EVENTS=() -} -is_parity_running() { - grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null + info "Stopping Docker containers..." + if command -v docker >/dev/null 2>&1; then + docker ps -q | xargs -r docker stop >/dev/null 2>&1 + fi + + info "Stopping User Scripts..." + pkill -f "/tmp/user.scripts" 2>/dev/null || true + + info "Syncing disks..." + sync + + echo "" + echo "$ICON_REBOOT_SMART Rebooting system NOW..." + sleep 5 + /sbin/reboot } # ----------------------------------------------------------------------------------------------- @@ -262,7 +284,7 @@ WATCHDOG_RUNNING=true cleanup() { echo "" - info "Docker watchdog received shutdown signal — stopping cleanly" + info "System watchdog received shutdown signal — stopping cleanly" WATCHDOG_RUNNING=false exit 0 } @@ -272,974 +294,140 @@ trap cleanup SIGTERM SIGINT # ----------------------------------------------------------------------------------------------- # ━━━ CONTINUOUS MONITORING LOOP ━━━ # ----------------------------------------------------------------------------------------------- -info "Docker watchdog started — checking every ${DOCKER_WATCHDOG_INTERVAL}s" +info "System watchdog started — checking every ${SYSTEM_WATCHDOG_INTERVAL}s" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" CYCLE=0 while [[ "$WATCHDOG_RUNNING" == true ]]; do ((CYCLE++)) - CYCLE_START=$(date +%s) - # Re-source Master.conf each cycle — picks up any config changes without restart + # Re-source Master.conf each cycle — picks up config changes without restart source "$SCRIPT_DIR/../Master.conf" + SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 )) + TOTAL_CORES=$(nproc) - # Rebuild per-host lists after re-source in case they changed - if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}") - for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do - WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}" - done - else - WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}") - for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do - WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}" - done + # Per-cycle triggers — cleared each iteration + TRIGGERS=() + + # ── rootfs usage ───────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then + ROOTFS_USED=$(df / --output=pcent | tail -1 | tr -d ' %') + TRIGGERED=false + [[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true + run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \ + TRIGGERS+=("rootfs=${ROOTFS_USED}%") fi - # Per-cycle variables — cleared each iteration, no accumulation - NOTIFY_EVENTS=() - T1_RESTARTS=0 - T1_WARNINGS=0 - T2_RESTARTS=0 - T2_WARNINGS=0 - - # Rebuild ignore map each cycle in case config was updated - declare -A IGNORE_MAP - for c in "${WATCHDOG_SCAN_IGNORE[@]}"; do - [[ -n "$c" ]] && IGNORE_MAP["$c"]=1 - done - - # Startup grace check - UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime) - IN_GRACE_PERIOD=false - if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then - IN_GRACE_PERIOD=true + # ── /var/log usage ──────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then + LOG_USED=$(df -P /var/log | awk 'NR==2 {print $5}' | tr -d '%') + TRIGGERED=false + [[ "$LOG_USED" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true + run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \ + TRIGGERS+=("log=${LOG_USED}%") fi - # Parity check — skip restarts during parity to avoid I/O interference - if is_parity_running; then - log "Parity check in progress — skipping restart actions this cycle" - sleep "$DOCKER_WATCHDOG_INTERVAL" - continue + # ── Free RAM ───────────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then + MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) + MEM_GB=$((MEM_KB / 1024 / 1024)) + TRIGGERED=false + [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]] && TRIGGERED=true + run_strike_check "ram" "$TRIGGERED" "RAM ${MEM_GB}GB free" && \ + TRIGGERS+=("low_ram=${MEM_GB}GB") fi - # ── TIER 1 — Strict Monitoring ────────────────────────────────────────────────────────── + # ── ZFS ARC ────────────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then + ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX )) + TRIGGERED=false + if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then + sync; echo 3 > /proc/sys/vm/drop_caches; sleep 5 + ARC_AFTER=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX )) + [[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]] && TRIGGERED=true + fi + run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned" && \ + TRIGGERS+=("arc_pinned=${ARC_PCT}%") + fi - # Required containers - if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then - for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do + # ── CPU temperature ────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then + CPU_TEMP="" + if command -v sensors >/dev/null 2>&1; then + CPU_TEMP=$(sensors 2>/dev/null | grep -i "Package id 0\|Tctl\|CPU Temp" | \ + awk '{print $NF}' | tr -d '+°C' | head -1) + fi + if [[ -n "$CPU_TEMP" ]]; then + CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP") + TRIGGERED=false + [[ "$CPU_TEMP_INT" -ge "$SYS_WATCHDOG_CPU_TEMP_MAX" ]] && TRIGGERED=true + run_strike_check "cpu_temp" "$TRIGGERED" "CPU temp ${CPU_TEMP_INT}°C" && \ + TRIGGERS+=("cpu_temp=${CPU_TEMP_INT}C") + fi + fi + + # ── Load average ───────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_LOAD" == true ]]; then + LOAD=$(awk '{print $1}' /proc/loadavg) + LOAD_INT=$(printf "%.0f" "$LOAD") + LOAD_THRESHOLD=$(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) + TRIGGERED=false + [[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true + run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \ + TRIGGERS+=("load=${LOAD}") + fi + + # ── Zombie processes ───────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then + ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" || echo 0) + TRIGGERED=false + [[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true + run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \ + TRIGGERS+=("zombies=${ZOMBIE_COUNT}") + fi + + # ── Docker daemon ──────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then + TRIGGERED=false + ! timeout 10 docker ps >/dev/null 2>&1 && TRIGGERED=true + run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon unresponsive" && \ + TRIGGERS+=("docker_daemon") + fi + + # ── Required containers from skip list ─────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then + FAILED_CONTAINERS=() + while IFS= read -r container; do [[ -z "$container" ]] && continue - STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") + [[ "$STATUS" != "true" ]] && FAILED_CONTAINERS+=("$container") + done < "$SYS_WATCHDOG_FAILED_FILE" - if is_skipped "$container"; then - if [[ "$STATUS" == "true" ]]; then - remove_from_skip_list "$container" - set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" - sed -i "/^${container}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null - queue_notify "$container recovered on $(hostname) — removed from skip list" "normal" - else - warn "$container — on skip list, manual intervention needed" - fi - continue - fi - - if [[ "$STATUS" == "true" ]]; then - set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" - else - STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE") - STRIKES=$(( STRIKES + 1 )) - set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE" - warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)" - ((T1_WARNINGS++)) - - if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then - result=0 - safe_restart "$container" "required container down" || result=$? - case $result in - 0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" - ((T1_RESTARTS++)) - queue_notify "$container was down and restarted on $(hostname)" "warning" ;; - 2) : ;; - *) queue_notify "$container failed to restart on $(hostname)" "warning" ;; - esac - fi - fi - done - fi - - # Memory and CPU monitoring - if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then - STATS=$(docker stats --no-stream \ - --format "{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}" 2>/dev/null) - TOTAL_CORES=$(nproc 2>/dev/null || echo 1) - - for container in "${!WATCHDOG_CONTAINERS[@]}"; do - MEM_LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}" - CONTAINER_STATS=$(echo "$STATS" | grep "^${container}|" | head -1) - [[ -z "$CONTAINER_STATS" ]] && continue - - MEM_USAGE=$(echo "$CONTAINER_STATS" | cut -d'|' -f2 | awk '{print $1}') - MEM_UNIT=$(echo "$MEM_USAGE" | grep -oE '[A-Za-z]+') - MEM_VALUE=$(echo "$MEM_USAGE" | grep -oE '[0-9.]+') - - case "$MEM_UNIT" in - GiB|GB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE * 1024}") ;; - MiB|MB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE}") ;; - KiB|KB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE / 1024}") ;; - *) MEM_MB=0 ;; - esac - - CPU_RAW=$(echo "$CONTAINER_STATS" | cut -d'|' -f3 | tr -d '%') - CPU_NORM=$(awk "BEGIN {printf \"%.1f\", $CPU_RAW / $TOTAL_CORES}") - CPU_INT=$(printf "%.0f" "$CPU_NORM") - - if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then - error "$container — memory exceeded hard limit ${MEM_LIMIT_MB}MB" - safe_restart "$container" "memory hard limit exceeded" - ((T1_RESTARTS++)) - queue_notify "$container exceeded memory limit on $(hostname) — restarted" "warning" - fi - - if [[ "$CPU_INT" -ge "$HARD_CPU_THRESHOLD" ]]; then - CPU_STRIKES=$(get_strikes "${container}_cpu" "$WATCHDOG_STATE_FILE") - CPU_STRIKES=$(( CPU_STRIKES + 1 )) - set_strikes "${container}_cpu" "$CPU_STRIKES" "$WATCHDOG_STATE_FILE" - if [[ "$CPU_STRIKES" -ge "$CPU_FAIL_LIMIT" ]]; then - safe_restart "$container" "CPU threshold exceeded" - set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE" - ((T1_RESTARTS++)) - queue_notify "$container CPU ${CPU_NORM}% on $(hostname) — restarted" "warning" - fi - else - set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE" - fi - done - fi - - # HTTP responsiveness - if [[ ${#WATCHDOG_CONTAINER_URLS[@]} -gt 0 ]]; then - for container in "${!WATCHDOG_CONTAINER_URLS[@]}"; do - URL="${WATCHDOG_CONTAINER_URLS[$container]}" - if curl -sf --max-time "$CURL_TIMEOUT" "$URL" >/dev/null 2>&1; then - set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE" - else - HTTP_STRIKES=$(get_strikes "${container}_http" "$WATCHDOG_STATE_FILE") - HTTP_STRIKES=$(( HTTP_STRIKES + 1 )) - set_strikes "${container}_http" "$HTTP_STRIKES" "$WATCHDOG_STATE_FILE" - warn "$container — not responding at $URL (strike $HTTP_STRIKES/$RESP_FAIL_LIMIT)" - ((T1_WARNINGS++)) - if [[ "$HTTP_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then - safe_restart "$container" "HTTP unresponsive" - set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE" - ((T1_RESTARTS++)) - queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning" - fi - fi - done - fi - - # ── TIER 2 — Global Health Scan ───────────────────────────────────────────────────────── - if [[ "$WATCHDOG_SCAN_ALL" == "true" ]]; then - ALL_CONTAINERS=$(docker ps --format "{{.Names}}" 2>/dev/null) - - # Unhealthy containers - if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then - UNHEALTHY=$(docker ps --filter health=unhealthy --format "{{.Names}}" 2>/dev/null) - while IFS= read -r container; do - [[ -z "$container" ]] && continue - [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue - is_skipped "$container" && continue - error "$container — unhealthy" - ((T2_WARNINGS++)) - result=0 - safe_restart "$container" "unhealthy health status" || result=$? - [[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \ - queue_notify "$container unhealthy on $(hostname) — restarted" "warning" - done <<< "$UNHEALTHY" - fi - - # OOM killed - if [[ "$WATCHDOG_NOTIFY_OOM" == "true" ]]; then - while IFS= read -r container; do - [[ -z "$container" ]] && continue - [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue - is_skipped "$container" && continue - OOM=$(docker inspect -f '{{.State.OOMKilled}}' "$container" 2>/dev/null) - if [[ "$OOM" == "true" ]]; then - error "$container — OOM killed" - ((T2_WARNINGS++)) - result=0 - safe_restart "$container" "OOM killed" || result=$? - [[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \ - queue_notify "$container OOM killed on $(hostname) — restarted" "warning" - fi - done <<< "$ALL_CONTAINERS" - fi - - # Crash loop detection - if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then - while IFS= read -r container; do - [[ -z "$container" ]] && continue - [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue - is_skipped "$container" && continue - RESTART_COUNT=$(docker inspect -f '{{.RestartCount}}' "$container" 2>/dev/null || echo 0) - PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d: -f2 || echo 0) - set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE" - if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then - ((T2_WARNINGS++)) - if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then - error "$container — crash loop CRITICAL: $RESTART_COUNT restarts" - queue_notify "$container crash loop CRITICAL on $(hostname) — manual intervention needed" "critical" - else - warn "$container — restarted since last check (total: $RESTART_COUNT)" - queue_notify "$container restarted on $(hostname) — count: $RESTART_COUNT" "warning" - fi - fi - done <<< "$ALL_CONTAINERS" - fi - - # Dead containers - if [[ "$WATCHDOG_RESTART_DEAD" == "true" ]]; then - DEAD=$(docker ps -a --filter status=dead --format "{{.Names}}" 2>/dev/null) - while IFS= read -r container; do - [[ -z "$container" ]] && continue - [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue - is_skipped "$container" && continue - error "$container — dead" - ((T2_WARNINGS++)) - RESTART_COUNT=$(get_restart_count "$container") - if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then - add_to_skip_list "$container" "dead — restarted $RESTART_COUNT times" - elif [[ "$DRY_RUN" == false ]]; then - docker rm "$container" >/dev/null 2>&1 - if docker start "$container" >/dev/null 2>&1; then - success "$container removed from dead state and restarted" - log_restart "$container" - ((T2_RESTARTS++)) - queue_notify "$container was dead on $(hostname) — restarted" "warning" - fi - fi - done <<< "$DEAD" - fi - - # Unexpected exits - if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then - CRASHED=$(docker ps -a \ - --filter status=exited \ - --format "{{.Names}}|{{.Status}}" 2>/dev/null | \ - grep -v "Exited (0)") - while IFS='|' read -r container status; do - [[ -z "$container" ]] && continue - [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue - is_skipped "$container" && continue - SKIP=false - for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do - [[ "$container" == "$req" ]] && SKIP=true && break - done - [[ "$SKIP" == true ]] && continue - error "$container — $status (unexpected exit)" - ((T2_WARNINGS++)) - result=0 - safe_restart "$container" "unexpected exit" || result=$? - [[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \ - queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning" - done <<< "$CRASHED" + if [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]]; then + run_strike_check "failed_containers" "true" "required containers stopped" && \ + TRIGGERS+=("containers=${FAILED_CONTAINERS[*]}") fi fi - # Send notifications if any events this cycle - flush_notify - - # Only log summary if something happened — quiet when all healthy - TOTAL_RESTARTS=$(( T1_RESTARTS + T2_RESTARTS )) - TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS )) - - if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then - CYCLE_END=$(date +%s) + # ── Evaluate triggers ──────────────────────────────────────────────────────────────────── + if [[ ${#TRIGGERS[@]} -gt 0 ]]; then echo "" - echo "━━━ $ICON_WATCHDOG Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━" - echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings" - echo "$ICON_WATCHDOG T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings" - echo "$ICON_TIME Duration: $(format_duration $(( CYCLE_END - CYCLE_START )))" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "━━━ $ICON_REBOOT_SMART System Watchdog — Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + for t in "${TRIGGERS[@]}"; do + echo " $ICON_REBOOT_SMART $t" + done + echo "" + do_reboot "${TRIGGERS[@]}" else - log "Cycle $CYCLE — all healthy ($(date '+%H:%M:%S'))" + log "Cycle $CYCLE — system healthy ($(date '+%H:%M:%S'))" fi # Sleep until next cycle — interruptible by SIGTERM - sleep "$DOCKER_WATCHDOG_INTERVAL" & + sleep "$SYSTEM_WATCHDOG_INTERVAL" & wait $! -done -# -# Tier 1 — Strict monitoring (configured containers only) -# Memory hard limits — immediate restart if exceeded -# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT strikes -# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT strikes -# Required containers — must always be running, strike system with skip list -# -# Tier 2 — Global health scan (all running containers) -# Unhealthy status — Docker HEALTHCHECK unhealthy → restart -# OOM killed — kernel killed container → restart + notify -# Crash loop detection — RestartCount climbing → notify, critical above limit -# Dead containers — remove and restart -# Unexpected exits — non-zero exit code → restart -# -# Cross-cutting intelligence (applies to both tiers): -# Startup grace period — skip restarts while system is still booting -# Dependency ordering — restart database before app, not the other way around -# Restart loop protect — stop restarting after X restarts in X hours → skip list -# Skip list auto-clear — clears when container recovers, fresh start -# Notification batching — one clean summary per run, not one ping per event -# -# State files: -# WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot) -# SYS_WATCHDOG_FAILED_FILE — persistent skip list (/boot — survives reboots) -# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (/boot) -# -# All configuration in Master.conf under Docker Watchdog section. -# Supports --dry-run and --status. -# ----------------------------------------------------------------------------------------------- - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" - -parse_args "$@" - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - -if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" - exit 1 -fi - -success "Running as root" - -acquire_lock - -# Select correct per-host watchdog lists -detect_hosts - -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}") - declare -A WATCHDOG_CONTAINER_URLS - for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do - WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}" - done -else - WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}") - declare -A WATCHDOG_CONTAINER_URLS - for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do - WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}" - done -fi - -info "Watchdog running as: $LOCAL_SERVER_NAME" -if ! command -v docker >/dev/null 2>&1; then - error "Docker not found" - exit 1 -fi - -success "Docker found" -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" - -# State files -touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \ - "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null - -# Build ignore map for Tier 2 -declare -A IGNORE_MAP -for c in "${WATCHDOG_SCAN_IGNORE[@]}"; do - [[ -n "$c" ]] && IGNORE_MAP["$c"]=1 -done - -# Batch notification collector -NOTIFY_EVENTS=() - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- -if [[ "$SHOW_STATUS" == true ]]; then - echo "" - echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_CONTAINERS Watched: ${!WATCHDOG_CONTAINERS[@]}" - echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}" - echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL" - echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s" - echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" - echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY" - echo "$ICON_WATCHDOG Ignore list: ${WATCHDOG_SCAN_IGNORE[*]:-none}" - echo "$ICON_GEAR Dry Run: $DRY_RUN" - UPTIME_S=$(awk '{print int($1)}' /proc/uptime) - echo "$ICON_TIME System uptime: $(format_duration $UPTIME_S)" - [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]] && \ - warn "Within startup grace period — restarts suppressed" - echo "━━━━━━━━━━━━━━━━━━━━━━━" - exit 0 -fi - -# ----------------------------------------------------------------------------------------------- -# HELPERS -# ----------------------------------------------------------------------------------------------- - -# Strike count management -get_strikes() { - grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0" -} - -set_strikes() { - local container="$1" count="$2" file="$3" - if grep -q "^${container}:" "$file" 2>/dev/null; then - sed -i "s/^${container}:.*/${container}:${count}/" "$file" - else - echo "${container}:${count}" >> "$file" - fi -} - -# Persistent skip list management -is_skipped() { - grep -q "^${1}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null -} - -add_to_skip_list() { - local container="$1" reason="$2" - if ! is_skipped "$container"; then - echo "$container" >> "$SYS_WATCHDOG_FAILED_FILE" - error "$container added to skip list — $reason" - queue_notify "$container added to skip list on $(hostname) — $reason — manual intervention needed" "critical" - fi -} - -remove_from_skip_list() { - sed -i "/^${1}$/d" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null - success "$1 removed from skip list — recovered" -} - -# Restart loop tracking — /boot/ bounded file, auto-purges old entries -log_restart() { - local container="$1" - local now - now=$(date '+%Y-%m-%d %H:%M:%S') - local cutoff - cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S') - - # Append this restart - echo "${container}|${now}" >> "$WATCHDOG_CONTAINER_RESTART_LOG" - - # Purge entries older than window — keeps file bounded - local tmp="${WATCHDOG_CONTAINER_RESTART_LOG}.tmp" - awk -F'|' -v cutoff="$cutoff" '$2 >= cutoff' \ - "$WATCHDOG_CONTAINER_RESTART_LOG" > "$tmp" && \ - mv "$tmp" "$WATCHDOG_CONTAINER_RESTART_LOG" -} - -# Count restarts for a container within the current window -get_restart_count() { - local container="$1" - local cutoff - cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S') - awk -F'|' -v c="$container" -v cutoff="$cutoff" \ - '$1==c && $2>=cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l -} - -# Dependency check — returns 0 if all dependencies are running -dependencies_satisfied() { - local container="$1" - local deps="${WATCHDOG_DEPENDENCIES[$container]:-}" - [[ -z "$deps" ]] && return 0 - - for dep in $deps; do - local status - status=$(docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null) - if [[ "$status" != "true" ]]; then - warn "$container — dependency $dep is not running — skipping restart this cycle" - return 1 - fi - done - return 0 -} - -# Restart a container with restart loop protection -# Returns 0 on success, 1 on failure, 2 if loop limit hit -safe_restart() { - local container="$1" reason="$2" - - # Check restart loop - local restart_count - restart_count=$(get_restart_count "$container") - - if [[ "$restart_count" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then - add_to_skip_list "$container" \ - "restarted $restart_count times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" - return 2 - fi - - # Check dependencies - dependencies_satisfied "$container" || return 1 - - # Check startup grace - local uptime_s - uptime_s=$(awk '{print int($1)}' /proc/uptime) - if [[ "$uptime_s" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then - warn "$container — within startup grace period (${uptime_s}s / ${WATCHDOG_STARTUP_GRACE}s) — skipping restart" - return 1 - fi - - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would restart $container ($reason)" - return 0 - fi - - info "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]..." - if docker restart "$container" >/dev/null 2>&1; then - success "$ICON_STARTED $container restarted" - log_restart "$container" - return 0 - else - error "Failed to restart $container" - return 1 - fi -} - -# Queue a notification event for batching -queue_notify() { - local message="$1" severity="${2:-warning}" - NOTIFY_EVENTS+=("${severity}|${message}") - log "Queued: $message" -} - -# Send all queued notifications -flush_notify() { - [[ ${#NOTIFY_EVENTS[@]} -eq 0 ]] && return - - if [[ "$WATCHDOG_BATCH_NOTIFY" == "true" ]]; then - # Build single batched message - local critical_count=0 warning_count=0 - local messages=() - local highest_severity="normal" - - for event in "${NOTIFY_EVENTS[@]}"; do - local sev="${event%%|*}" - local msg="${event#*|}" - messages+=("$msg") - [[ "$sev" == "critical" ]] && ((critical_count++)) && highest_severity="warning" - [[ "$sev" == "warning" ]] && ((warning_count++)) && \ - [[ "$highest_severity" == "normal" ]] && highest_severity="warning" - done - - local summary - summary=$(printf '%s. ' "${messages[@]}") - notify "Docker Watchdog on $(hostname) — ${#NOTIFY_EVENTS[@]} event(s): $summary" \ - "Docker Watchdog" "$highest_severity" - else - # Send individual notifications - for event in "${NOTIFY_EVENTS[@]}"; do - local sev="${event%%|*}" - local msg="${event#*|}" - [[ "$sev" == "critical" ]] && sev="warning" - notify "$msg" "Docker Watchdog" "$sev" - done - fi - - NOTIFY_EVENTS=() -} - -# ----------------------------------------------------------------------------------------------- -# STARTUP GRACE CHECK -# ----------------------------------------------------------------------------------------------- -UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime) -IN_GRACE_PERIOD=false - -if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then - GRACE_REMAINING=$(( WATCHDOG_STARTUP_GRACE - UPTIME_SECONDS )) - warn "System uptime $(format_duration $UPTIME_SECONDS) — startup grace period active (${GRACE_REMAINING}s remaining)" - warn "Checks will run but restarts are suppressed during grace period" - IN_GRACE_PERIOD=true -else - info "System uptime $(format_duration $UPTIME_SECONDS) — grace period passed" -fi - -# ----------------------------------------------------------------------------------------------- -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# TIER 1 — Strict Monitoring -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ----------------------------------------------------------------------------------------------- - -echo "" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo " $ICON_WATCHDOG TIER 1 — Strict Monitoring" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -T1_RESTARTS=0 -T1_WARNINGS=0 - -# ── Required Containers ────────────────────────────────────────────────────────────────────── -if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then - echo "" - echo "━━━ $ICON_CONTAINERS Required Containers ━━━" - - for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do - [[ -z "$container" ]] && continue - - STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) - - # Check skip list - if is_skipped "$container"; then - if [[ "$STATUS" == "true" ]]; then - remove_from_skip_list "$container" - set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" - # Clear restart history on recovery - sed -i "/^${container}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null - queue_notify "$container recovered on $(hostname) — removed from skip list" "normal" - else - warn "$ICON_NOT_RUNNING $container — on skip list, manual intervention needed" - RESTART_COUNT=$(get_restart_count "$container") - warn " Restarted $RESTART_COUNT time(s) in last ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" - fi - continue - fi - - if [[ "$STATUS" == "true" ]]; then - success "$ICON_RUNNING $container — running" - set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" - else - STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE") - STRIKES=$(( STRIKES + 1 )) - set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE" - warn "$ICON_NOT_RUNNING $container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)" - ((T1_WARNINGS++)) - - if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then - result=0 - safe_restart "$container" "required container down" || result=$? - case $result in - 0) - set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" - ((T1_RESTARTS++)) - queue_notify "$container was down and restarted on $(hostname)" "warning" - ;; - 2) - : # already added to skip list in safe_restart - ;; - *) - queue_notify "$container failed to restart on $(hostname) — check dependencies and logs" "warning" - ;; - esac - fi - fi - done -fi - -# ── Memory and CPU Monitoring ──────────────────────────────────────────────────────────────── -if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then - echo "" - echo "━━━ $ICON_MEM Memory / CPU Monitoring ━━━" - - STATS=$(docker stats --no-stream \ - --format "{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}" 2>/dev/null) - TOTAL_CORES=$(nproc 2>/dev/null || echo 1) - - for container in "${!WATCHDOG_CONTAINERS[@]}"; do - MEM_LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}" - CONTAINER_STATS=$(echo "$STATS" | grep "^${container}|" | head -1) - [[ -z "$CONTAINER_STATS" ]] && continue - - MEM_USAGE=$(echo "$CONTAINER_STATS" | cut -d'|' -f2 | awk '{print $1}') - MEM_UNIT=$(echo "$MEM_USAGE" | grep -oE '[A-Za-z]+') - MEM_VALUE=$(echo "$MEM_USAGE" | grep -oE '[0-9.]+') - - case "$MEM_UNIT" in - GiB|GB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE * 1024}") ;; - MiB|MB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE}") ;; - KiB|KB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE / 1024}") ;; - *) MEM_MB=0 ;; - esac - - MEM_PCT=$(awk "BEGIN {printf \"%.0f\", $MEM_MB * 100 / $MEM_LIMIT_MB}") - CPU_RAW=$(echo "$CONTAINER_STATS" | cut -d'|' -f3 | tr -d '%') - CPU_NORM=$(awk "BEGIN {printf \"%.1f\", $CPU_RAW / $TOTAL_CORES}") - CPU_INT=$(printf "%.0f" "$CPU_NORM") - - # Memory check - if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then - error "$container — memory ${MEM_VALUE}${MEM_UNIT} exceeded hard limit ${MEM_LIMIT_MB}MB" - safe_restart "$container" "memory hard limit exceeded" - ((T1_RESTARTS++)) - queue_notify "$container exceeded memory limit on $(hostname) — ${MEM_VALUE}${MEM_UNIT} / ${MEM_LIMIT_MB}MB — restarted" "warning" - elif [[ "$MEM_PCT" -ge "$SOFT_MEM_THRESHOLD" ]]; then - warn "$container — memory ${MEM_VALUE}${MEM_UNIT} (${MEM_PCT}% of ${MEM_LIMIT_MB}MB)" - ((T1_WARNINGS++)) - else - success "$container — memory ${MEM_VALUE}${MEM_UNIT} (${MEM_PCT}%) CPU ${CPU_NORM}%" - fi - - # CPU check - if [[ "$CPU_INT" -ge "$HARD_CPU_THRESHOLD" ]]; then - CPU_STRIKES=$(get_strikes "${container}_cpu" "$WATCHDOG_STATE_FILE") - CPU_STRIKES=$(( CPU_STRIKES + 1 )) - set_strikes "${container}_cpu" "$CPU_STRIKES" "$WATCHDOG_STATE_FILE" - warn "$container — CPU ${CPU_NORM}% (strike $CPU_STRIKES/$CPU_FAIL_LIMIT)" - if [[ "$CPU_STRIKES" -ge "$CPU_FAIL_LIMIT" ]]; then - safe_restart "$container" "CPU threshold exceeded" - set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE" - ((T1_RESTARTS++)) - queue_notify "$container CPU ${CPU_NORM}% on $(hostname) — restarted after $CPU_FAIL_LIMIT strikes" "warning" - fi - else - set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE" - fi - done -fi - -# ── HTTP Responsiveness ────────────────────────────────────────────────────────────────────── -if [[ ${#WATCHDOG_CONTAINER_URLS[@]} -gt 0 ]]; then - echo "" - echo "━━━ $ICON_NET HTTP Responsiveness ━━━" - - for container in "${!WATCHDOG_CONTAINER_URLS[@]}"; do - URL="${WATCHDOG_CONTAINER_URLS[$container]}" - if curl -sf --max-time "$CURL_TIMEOUT" "$URL" >/dev/null 2>&1; then - success "$container — responding at $URL" - set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE" - else - HTTP_STRIKES=$(get_strikes "${container}_http" "$WATCHDOG_STATE_FILE") - HTTP_STRIKES=$(( HTTP_STRIKES + 1 )) - set_strikes "${container}_http" "$HTTP_STRIKES" "$WATCHDOG_STATE_FILE" - warn "$container — not responding at $URL (strike $HTTP_STRIKES/$RESP_FAIL_LIMIT)" - ((T1_WARNINGS++)) - if [[ "$HTTP_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then - safe_restart "$container" "HTTP unresponsive" - set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE" - ((T1_RESTARTS++)) - queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning" - fi - fi - done -fi - -# ----------------------------------------------------------------------------------------------- -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# TIER 2 — Global Health Scan -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ----------------------------------------------------------------------------------------------- - -T2_RESTARTS=0 -T2_WARNINGS=0 - -if [[ "$WATCHDOG_SCAN_ALL" != "true" ]]; then - info "Global scan disabled (WATCHDOG_SCAN_ALL=false) — skipping Tier 2" -else - echo "" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo " $ICON_WATCHDOG TIER 2 — Global Health Scan" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - - ALL_CONTAINERS=$(docker ps --format "{{.Names}}" 2>/dev/null) - - # ── Unhealthy containers ───────────────────────────────────────────────────────────────── - if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then - echo "" - echo "━━━ $ICON_HEALTH Unhealthy Status ━━━" - UNHEALTHY=$(docker ps --filter health=unhealthy --format "{{.Names}}" 2>/dev/null) - - if [[ -z "$UNHEALTHY" ]]; then - success "No unhealthy containers" - else - while IFS= read -r container; do - [[ -z "$container" ]] && continue - [[ -n "${IGNORE_MAP[$container]:-}" ]] && \ - { info "Skipping $container (ignore list)"; continue; } - is_skipped "$container" && \ - { warn "$container — unhealthy but on skip list, manual intervention needed"; continue; } - - error "$container — Docker health status: unhealthy" - ((T2_WARNINGS++)) - result=0 - safe_restart "$container" "unhealthy health status" || result=$? - if [[ $result -eq 0 ]]; then - ((T2_RESTARTS++)) - queue_notify "$container unhealthy on $(hostname) — restarted" "warning" - elif [[ $result -eq 2 ]]; then - : # skip list message already queued - fi - done <<< "$UNHEALTHY" - fi - fi - - # ── OOM killed containers ──────────────────────────────────────────────────────────────── - if [[ "$WATCHDOG_NOTIFY_OOM" == "true" ]]; then - echo "" - echo "━━━ $ICON_MEM OOM Kill Check ━━━" - OOM_FOUND=false - - while IFS= read -r container; do - [[ -z "$container" ]] && continue - [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue - is_skipped "$container" && continue - - OOM=$(docker inspect -f '{{.State.OOMKilled}}' "$container" 2>/dev/null) - if [[ "$OOM" == "true" ]]; then - error "$container — OOM killed by kernel" - OOM_FOUND=true - ((T2_WARNINGS++)) - result=0 - safe_restart "$container" "OOM killed" || result=$? - if [[ $result -eq 0 ]]; then - ((T2_RESTARTS++)) - queue_notify "$container OOM killed on $(hostname) — system under memory pressure — restarted" "warning" - fi - fi - done <<< "$ALL_CONTAINERS" - - [[ "$OOM_FOUND" == false ]] && success "No OOM kills detected" - fi - - # ── Crash loop detection ───────────────────────────────────────────────────────────────── - if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then - echo "" - echo "━━━ $ICON_REBOOT_SMART Crash Loop Detection ━━━" - CRASH_FOUND=false - - while IFS= read -r container; do - [[ -z "$container" ]] && continue - [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue - is_skipped "$container" && continue - - RESTART_COUNT=$(docker inspect -f '{{.RestartCount}}' "$container" 2>/dev/null || echo 0) - PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d: -f2 || echo 0) - - # Update baseline - set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE" - - if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then - CRASH_FOUND=true - ((T2_WARNINGS++)) - if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then - error "$container — crash loop CRITICAL: $RESTART_COUNT total Docker restarts" - queue_notify "$container crash loop CRITICAL on $(hostname) — $RESTART_COUNT Docker restarts — manual intervention needed" "critical" - else - warn "$container — Docker restarted since last check (total: $RESTART_COUNT)" - queue_notify "$container restarted on $(hostname) — Docker restart count: $RESTART_COUNT" "warning" - fi - elif [[ "$RESTART_COUNT" -gt 0 ]]; then - log "$container — Docker restart count: $RESTART_COUNT (stable)" - fi - done <<< "$ALL_CONTAINERS" - - [[ "$CRASH_FOUND" == false ]] && success "No crash loops detected" - fi - - # ── Dead containers ────────────────────────────────────────────────────────────────────── - if [[ "$WATCHDOG_RESTART_DEAD" == "true" ]]; then - echo "" - echo "━━━ $ICON_STOPPED Dead Containers ━━━" - DEAD=$(docker ps -a --filter status=dead --format "{{.Names}}" 2>/dev/null) - - if [[ -z "$DEAD" ]]; then - success "No dead containers" - else - while IFS= read -r container; do - [[ -z "$container" ]] && continue - [[ -n "${IGNORE_MAP[$container]:-}" ]] && \ - { info "Skipping $container (ignore list)"; continue; } - is_skipped "$container" && \ - { warn "$container — dead but on skip list, manual intervention needed"; continue; } - - error "$container — status: dead" - ((T2_WARNINGS++)) - RESTART_COUNT=$(get_restart_count "$container") - if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then - add_to_skip_list "$container" "dead state — restarted $RESTART_COUNT times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" - elif [[ "$DRY_RUN" == false ]]; then - docker rm "$container" >/dev/null 2>&1 - if docker start "$container" >/dev/null 2>&1; then - success "$container removed from dead state and restarted" - log_restart "$container" - ((T2_RESTARTS++)) - queue_notify "$container was dead on $(hostname) — removed and restarted" "warning" - else - error "$container failed to restart after dead state" - queue_notify "$container dead and failed to restart on $(hostname) — manual intervention needed" "warning" - fi - else - warn "DRY RUN — would remove and restart $container" - fi - done <<< "$DEAD" - fi - fi - - # ── Unexpected exits ───────────────────────────────────────────────────────────────────── - if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then - echo "" - echo "━━━ $ICON_ERROR Unexpected Exits ━━━" - CRASHED=$(docker ps -a \ - --filter status=exited \ - --format "{{.Names}}|{{.Status}}" 2>/dev/null | \ - grep -v "Exited (0)") - - if [[ -z "$CRASHED" ]]; then - success "No unexpected exits" - else - while IFS='|' read -r container status; do - [[ -z "$container" ]] && continue - [[ -n "${IGNORE_MAP[$container]:-}" ]] && \ - { info "Skipping $container (ignore list)"; continue; } - is_skipped "$container" && \ - { warn "$container — crashed but on skip list, manual intervention needed"; continue; } - - # Skip Tier 1 required containers — already handled above - SKIP=false - for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do - [[ "$container" == "$req" ]] && SKIP=true && break - done - [[ "$SKIP" == true ]] && continue - - error "$container — $status (unexpected exit)" - ((T2_WARNINGS++)) - result=0 - safe_restart "$container" "unexpected exit" || result=$? - if [[ $result -eq 0 ]]; then - ((T2_RESTARTS++)) - queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning" - elif [[ $result -eq 2 ]]; then - : # skip list message already queued - fi - done <<< "$CRASHED" - fi - fi -fi - -# ----------------------------------------------------------------------------------------------- -# Send batched notifications -# ----------------------------------------------------------------------------------------------- -flush_notify - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- -TOTAL_RESTARTS=$(( T1_RESTARTS + T2_RESTARTS )) -TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS )) - -echo "" -echo "━━━━━ $ICON_SUMMARY DOCKER WATCHDOG SUMMARY ━━━━━" -echo "$ICON_WATCHDOG Tier 1 (strict): $T1_RESTARTS restarts / $T1_WARNINGS warnings" -echo "$ICON_WATCHDOG Tier 2 (global): $T2_RESTARTS restarts / $T2_WARNINGS warnings" -echo "$ICON_WATCHDOG Total: $TOTAL_RESTARTS restarts / $TOTAL_WARNINGS warnings" -[[ "$IN_GRACE_PERIOD" == "true" ]] && \ - echo "$ICON_WARN Grace period: active — restarts suppressed" -echo "" -if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" -elif [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then - echo "$ICON_WARN Status: $TOTAL_RESTARTS container(s) restarted / $TOTAL_WARNINGS warning(s)" -else - echo "$ICON_DONE Status: $ICON_SUCCESS ALL CONTAINERS HEALTHY" -fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +done \ No newline at end of file diff --git a/Master.conf b/Master.conf index 7b84e73..2ad9896 100644 --- a/Master.conf +++ b/Master.conf @@ -154,13 +154,13 @@ # Continuous scripts (watchdogs, failover) run until array stops. ARRAY_START_SCRIPTS=( - "unRAID_Essentials/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts + "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning - #"Docker_Essentials/docker_network_connect.sh" # connect containers to extra networks + "Docker_Essentials/docker_network_connect.sh" # connect containers to extra networks "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop - #"Failover/failover.sh" # mutual failover — continuous loop + "Failover/failover.sh" # mutual failover — continuous loop ) # ━━━ Daily Sync Maintenance ━━━ @@ -201,20 +201,6 @@ HOST2_DAILY_SYNC_SHARES=( /mnt/user/Anime_Shows ) -# Job list run directly by daily_sync_maintenance.sh after the media share sync. -# Runs sequentially — permissions first, then cleaners, then arr cleanup. -# Comment out any job to disable without removing it. -# Each individual script can still be run manually for one-off maintenance. - -MEDIA_MANAGEMENT_JOBS=( - "Media/media_shares_permissions.sh" # apply permissions — runs first - "Media/media_cleaner.sh anime" # remove junk from anime shares - "Media/media_cleaner.sh media" # remove junk from media shares - "Media/lidarr_cleanup.sh" # remove orphaned music files - "Media/sonarr_cleanup.sh" # remove orphaned TV files - "Media/radarr_cleanup.sh" # remove orphaned movie files -) - # Personal encrypted shares — synced for offsite backup, independent of media shares. # ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. # See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. @@ -250,7 +236,22 @@ WEEKLY_SYNC_JOBS=( # Both false → sync only, no updates. # Toggle false temporarily to skip updates without changing the schedule. CRITICAL_SYNC_UPDATES=true # pull container updates locally - CRITICAL_SYNC_UPDATES_REMOTE=false # pull container updates on remote via SSH + CRITICAL_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH + +# ━━━ Media Management ━━━ +# Job list run directly by daily_sync_maintenance.sh after the media share sync. +# Runs sequentially — permissions first, then cleaners, then arr cleanup. +# Comment out any job to disable without removing it. +# Each individual script can still be run manually for one-off maintenance. + +MEDIA_MANAGEMENT_JOBS=( + "Media/media_shares_permissions.sh" # apply permissions — runs first + "Media/media_cleaner.sh anime" # remove junk from anime shares + "Media/media_cleaner.sh media" # remove junk from media shares + "Media/lidarr_cleanup.sh" # remove orphaned music files + "Media/sonarr_cleanup.sh" # remove orphaned TV files + "Media/radarr_cleanup.sh" # remove orphaned movie files +) # ============================================================================================== # ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── diff --git a/common.sh b/common.sh index a1e3182..d329e26 100644 --- a/common.sh +++ b/common.sh @@ -6,7 +6,7 @@ # ----------------------------------------------------------------------------------------------- # Changelog: # v1.0 — Initial stable framework -# v1.1 — format_duration moved here from daily_sync.sh for shared use +# v1.1 — format_duration moved here from daily_sync_maintenance.sh for shared use # SSH_KEY collision resolved — gitea key renamed GITEA_SSH_KEY in Master.conf # Version and changelog tracking added # v1.2 — Consistent function header comment blocks across all functions @@ -92,6 +92,7 @@ ICON_STOPPED="🔴" # container confirmed stopped ICON_START="▶️" # start command being issued ICON_STARTED="💚" # container confirmed started ICON_RUNNING="🟢" # container already running when checked +ICON_SKIP="⏭️" # skipping — already running healthy instance ICON_NOT_RUNNING="⭕" # container already stopped when checked # Transfer @@ -695,6 +696,8 @@ _release_on_exit() { # acquire_lock — acquire exclusive lock for this script # Mode: strict (default) — exit immediately if locked # wait — wait LOCK_WAIT_TIMEOUT seconds then exit +# continuous — for long-running scripts: skip gracefully if healthy, +# clear and restart if dead/stuck # Stale lock: if PID in lock file is dead → clear and acquire # Age warning: if lock older than LOCK_WARN_AGE → warn # ----------------------------------------------------------------------------------------------- @@ -724,7 +727,19 @@ acquire_lock() { warn "$script_name has been running for ${lock_age}s — may be stuck (PID $existing_pid)" fi - if [[ "$mode" == "wait" ]]; then + if [[ "$mode" == "continuous" ]]; then + # Continuous scripts (watchdogs, failover) — healthy instance = skip gracefully + # Only force-restart if PID is stuck/unresponsive beyond LOCK_WARN_AGE + if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]]; then + warn "$script_name appears stuck (running ${lock_age}s) — clearing and restarting" + kill "$existing_pid" 2>/dev/null + sleep 2 + rm -f "$lockfile" + else + log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping" + exit 0 + fi + elif [[ "$mode" == "wait" ]]; then info "Another instance of $script_name is running — waiting up to ${LOCK_WAIT_TIMEOUT}s" local waited=0 while [[ -f "$lockfile" ]] && [[ "$waited" -lt "$LOCK_WAIT_TIMEOUT" ]]; do @@ -746,7 +761,7 @@ acquire_lock() { error "Another instance of $script_name is already running (PID $existing_pid) — exiting" # Notify for critical scripts that should rarely overlap case "$script_name" in - failover|transcode_management|media_management|daily_sync|system_watchdog) + failover|transcode_management|media_management|daily_sync_maintenance|system_watchdog) notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning" ;; esac diff --git a/unRAID_Essentials/system_watchdog.sh b/unRAID_Essentials/system_watchdog.sh index 4b32e01..a423d33 100644 --- a/unRAID_Essentials/system_watchdog.sh +++ b/unRAID_Essentials/system_watchdog.sh @@ -57,7 +57,7 @@ fi success "Running as root" -acquire_lock +acquire_lock "continuous" TOTAL_CORES=$(nproc) @@ -430,495 +430,4 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do sleep "$SYSTEM_WATCHDOG_INTERVAL" & wait $! -done -# -# Checks (all toggleable in Master.conf): -# rootfs usage — high rootfs fills rapidly when array is down, crash imminent -# /var/log usage — log spam can fill rootfs, indicates something is broken -# free RAM — critically low RAM means OOM or swap imminent -# ZFS ARC pinned — ARC not releasing after reclaim means memory is stuck -# CPU temperature — sustained tjmax causes throttling or kernel panic -# load average — sustained high load means something is stuck or runaway -# zombie processes — large zombie count indicates serious process management failure -# Docker daemon — unresponsive daemon means containers cannot be managed -# Required containers — stopped containers that should be running (after watchdog skip list) -# -# Abort conditions (toggleable — true = abort, false = reboot anyway): -# ZFS pool unhealthy — reboot with bad pool risks data loss -# Parity running — aborting parity is better than crashing mid-check -# Mover running — aborting move is better than crashing mid-move -# -# Reboot loop protection: -# Tracks reboot timestamps in persistent log on /boot/ -# Rolling window — old entries purge automatically after SYS_WATCHDOG_REBOOT_WINDOW_HRS -# If reboot count hits SYS_WATCHDOG_REBOOT_LIMIT in window → shutdown instead of reboot -# -# All configuration in Master.conf under System Watchdog section. -# Supports --dry-run to show triggered conditions without rebooting. -# ----------------------------------------------------------------------------------------------- - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" - -parse_args "$@" - -# Convert window hours to seconds for internal use -SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 )) - -TOTAL_CORES=$(nproc) - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - -if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" - exit 1 -fi - -success "Running as root" - -acquire_lock - -# Ensure state and persistent files exist -touch "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || { - error "Cannot create state file: $SYS_WATCHDOG_STATE_FILE" - exit 1 -} - -touch "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || { - error "Cannot create reboot log: $SYS_WATCHDOG_REBOOT_LOG" - exit 1 -} - -touch "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || { - error "Cannot create failed container list: $SYS_WATCHDOG_FAILED_FILE" - exit 1 -} - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- -if [[ "$SHOW_STATUS" == true ]]; then - echo "" - echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_HEALTH rootfs threshold: ${SYS_WATCHDOG_ROOTFS_PCT}% toggle: $SYS_WATCHDOG_CHECK_ROOTFS" - echo "$ICON_HEALTH log threshold: ${SYS_WATCHDOG_LOG_PCT}% toggle: $SYS_WATCHDOG_CHECK_LOG" - echo "$ICON_MEM RAM threshold: ${SYS_WATCHDOG_MEM_GB}GB free toggle: $SYS_WATCHDOG_CHECK_RAM" - echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}% toggle: $SYS_WATCHDOG_CHECK_ARC" - echo "$ICON_ZFS ARC release: ${SYS_WATCHDOG_ARC_RELEASE_PCT}%" - echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C toggle: $SYS_WATCHDOG_CHECK_CPU_TEMP" - echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x cores toggle: $SYS_WATCHDOG_CHECK_LOAD" - echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT} toggle: $SYS_WATCHDOG_CHECK_ZOMBIES" - echo "$ICON_CONTAINERS Docker daemon: toggle: $SYS_WATCHDOG_CHECK_DOCKER_DAEMON" - echo "$ICON_CONTAINERS Containers: toggle: $SYS_WATCHDOG_CHECK_CONTAINERS" - echo "$ICON_SHIELD Strike limit: $SYS_WATCHDOG_STRIKE_LIMIT" - echo "$ICON_REBOOT_SMART Reboot limit: $SYS_WATCHDOG_REBOOT_LIMIT in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs" - echo "$ICON_ZFS Abort ZFS unhealthy: $SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" - echo "$ICON_GEAR Abort on parity: $SYS_WATCHDOG_ABORT_ON_PARITY" - echo "$ICON_MOVER Abort on mover: $SYS_WATCHDOG_ABORT_ON_MOVER" - echo "$ICON_GEAR Dry Run: $DRY_RUN" - echo "━━━━━━━━━━━━━━━━━━━━━━━" - exit 0 -fi - -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboot will be executed" - -# ----------------------------------------------------------------------------------------------- -# STATE HELPERS -# ----------------------------------------------------------------------------------------------- - -get_strikes() { - local key="$1" - grep -E "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2 -} - -set_strikes() { - local key="$1" count="$2" - grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp" - echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp" - mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE" -} - -# Returns current strike count, increments and saves, then echoes new count -increment_strikes() { - local key="$1" - local current - current=$(get_strikes "$key") - [[ -z "$current" ]] && current=0 - ((current++)) - set_strikes "$key" "$current" - echo "$current" -} - -reset_strikes() { - local key="$1" - set_strikes "$key" 0 -} - -# ----------------------------------------------------------------------------------------------- -# REBOOT LOG HELPERS -# Tracks reboot timestamps for loop detection. -# Rolling window — entries older than SYS_WATCHDOG_REBOOT_WINDOW are purged automatically. -# ----------------------------------------------------------------------------------------------- - -purge_old_reboots() { - local now - now=$(date +%s) - local cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW )) - - grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do - [[ "$ts" -gt "$cutoff" ]] && echo "$ts" - done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp" - mv "${SYS_WATCHDOG_REBOOT_LOG}.tmp" "$SYS_WATCHDOG_REBOOT_LOG" -} - -count_recent_reboots() { - purge_old_reboots - grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0 -} - -log_reboot() { - date +%s >> "$SYS_WATCHDOG_REBOOT_LOG" -} - -# ----------------------------------------------------------------------------------------------- -# ABORT CONDITION CHECKS -# Run before any reboot is triggered — abort conditions prevent rebooting -# when it would make things worse than letting the system run. -# ----------------------------------------------------------------------------------------------- - -check_abort_conditions() { - local should_abort=false - - # ZFS pool health - if command -v zpool >/dev/null 2>&1; then - local unhealthy - unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true) - if [[ -n "$unhealthy" ]]; then - if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then - error "$ICON_ZFS ZFS pool unhealthy — aborting reboot (ABORT_ON_ZFS_UNHEALTHY=true)" - notify "System watchdog aborted reboot on $(hostname) — ZFS pool unhealthy" "System Watchdog" "warning" - should_abort=true - else - warn "$ICON_ZFS ZFS pool unhealthy — continuing reboot (ABORT_ON_ZFS_UNHEALTHY=false)" - fi - fi - fi - - # Parity check running - if [[ -f /var/local/emhttp/parity-date.txt ]]; then - if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then - if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then - error "$ICON_DISK Parity check running — aborting reboot (ABORT_ON_PARITY=true)" - notify "System watchdog aborted reboot on $(hostname) — parity check running" "System Watchdog" "warning" - should_abort=true - else - warn "$ICON_DISK Parity check running — continuing reboot (ABORT_ON_PARITY=false)" - fi - fi - fi - - # Mover running - if pgrep -f "mover" >/dev/null 2>&1; then - if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then - error "$ICON_MOVER Mover is running — aborting reboot (ABORT_ON_MOVER=true)" - notify "System watchdog aborted reboot on $(hostname) — mover running" "System Watchdog" "warning" - should_abort=true - else - warn "$ICON_MOVER Mover is running — continuing reboot (ABORT_ON_MOVER=false)" - fi - fi - - [[ "$should_abort" == true ]] && return 1 - return 0 -} - -# ----------------------------------------------------------------------------------------------- -# STRIKE-BASED CHECK HELPER -# Runs a check function, increments strikes on trigger, resets on clear. -# Returns 0 if strike limit hit (reboot trigger), 1 otherwise. -# Usage: run_strike_check "key" "triggered (true/false)" "description" -# ----------------------------------------------------------------------------------------------- -run_strike_check() { - local key="$1" triggered="$2" description="$3" - - if [[ "$triggered" == true ]]; then - local strikes - strikes=$(increment_strikes "$key") - warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT" - if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then - error "$description hit strike limit — reboot triggered" - reset_strikes "$key" - return 0 - fi - else - local current - current=$(get_strikes "$key") - if [[ -n "$current" && "$current" -gt 0 ]]; then - info "$description — recovered, resetting strikes" - reset_strikes "$key" - fi - fi - return 1 -} - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SHIELD Health Checks ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_SHIELD Health Checks — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "" - -TRIGGERS=() - -# rootfs usage -if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then - info "$ICON_HEALTH Checking rootfs usage..." - ROOTFS_USED=$(df / --output=pcent | tail -1 | tr -d ' %') - TRIGGERED=false - if [[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]]; then - error "$ICON_HEALTH rootfs is ${ROOTFS_USED}% full — threshold ${SYS_WATCHDOG_ROOTFS_PCT}%" - TRIGGERED=true - else - success "$ICON_HEALTH rootfs: ${ROOTFS_USED}% used" - fi - run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && TRIGGERS+=("rootfs=${ROOTFS_USED}%") -fi - -# /var/log usage -if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then - info "$ICON_HEALTH Checking /var/log usage..." - LOG_USED=$(df -P /var/log | awk 'NR==2 {print $5}' | tr -d '%') - TRIGGERED=false - if [[ "$LOG_USED" -ge "$SYS_WATCHDOG_LOG_PCT" ]]; then - error "$ICON_HEALTH /var/log is ${LOG_USED}% full — threshold ${SYS_WATCHDOG_LOG_PCT}%" - TRIGGERED=true - else - success "$ICON_HEALTH /var/log: ${LOG_USED}% used" - fi - run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && TRIGGERS+=("log=${LOG_USED}%") -fi - -# Free RAM -if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then - info "$ICON_MEM Checking available RAM..." - MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) - MEM_GB=$((MEM_KB / 1024 / 1024)) - TRIGGERED=false - if [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]]; then - error "$ICON_MEM Available RAM: ${MEM_GB}GB — threshold ${SYS_WATCHDOG_MEM_GB}GB" - TRIGGERED=true - else - success "$ICON_MEM Available RAM: ${MEM_GB}GB" - fi - run_strike_check "ram" "$TRIGGERED" "RAM ${MEM_GB}GB free" && TRIGGERS+=("low_ram=${MEM_GB}GB") -fi - -# ZFS ARC pinned -if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then - info "$ICON_ZFS Checking ZFS ARC..." - ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) - ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats) - ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX )) - TRIGGERED=false - - if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then - warn "$ICON_ZFS ARC at ${ARC_PCT}% — attempting reclaim..." - sync - echo 3 > /proc/sys/vm/drop_caches - sleep 5 - ARC_AFTER=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) - ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX )) - if [[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]]; then - error "$ICON_ZFS ARC still ${ARC_AFTER_PCT}% after reclaim — threshold ${SYS_WATCHDOG_ARC_RELEASE_PCT}%" - TRIGGERED=true - else - success "$ICON_ZFS ARC released to ${ARC_AFTER_PCT}% after reclaim" - fi - else - success "$ICON_ZFS ARC: ${ARC_PCT}% of max" - fi - run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned" && TRIGGERS+=("arc_pinned=${ARC_PCT}%") -elif [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]]; then - info "$ICON_ZFS ZFS arcstats not available — skipping" -fi - -# CPU temperature -if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then - info "$ICON_GEAR Checking CPU temperature..." - CPU_TEMP="" - if command -v sensors >/dev/null 2>&1; then - CPU_TEMP=$(sensors 2>/dev/null | grep -i "Package id 0\|Tctl\|CPU Temp" | awk '{print $NF}' | tr -d '+°C' | head -1) - fi - - if [[ -z "$CPU_TEMP" ]]; then - info "$ICON_GEAR CPU temperature sensor not available — skipping" - else - CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP") - TRIGGERED=false - if [[ "$CPU_TEMP_INT" -ge "$SYS_WATCHDOG_CPU_TEMP_MAX" ]]; then - error "$ICON_GEAR CPU temp ${CPU_TEMP_INT}°C — threshold ${SYS_WATCHDOG_CPU_TEMP_MAX}°C" - TRIGGERED=true - else - success "$ICON_GEAR CPU temp: ${CPU_TEMP_INT}°C" - fi - run_strike_check "cpu_temp" "$TRIGGERED" "CPU temp ${CPU_TEMP_INT}°C" && TRIGGERS+=("cpu_temp=${CPU_TEMP_INT}C") - fi -fi - -# Load average -if [[ "$SYS_WATCHDOG_CHECK_LOAD" == true ]]; then - info "$ICON_GEAR Checking load average..." - LOAD=$(awk '{print $1}' /proc/loadavg) - LOAD_INT=$(printf "%.0f" "$LOAD") - LOAD_THRESHOLD=$(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) - TRIGGERED=false - if [[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]]; then - error "$ICON_GEAR Load average ${LOAD} — threshold ${LOAD_THRESHOLD} (${TOTAL_CORES} cores x ${SYS_WATCHDOG_LOAD_MULTIPLIER})" - TRIGGERED=true - else - success "$ICON_GEAR Load average: ${LOAD}" - fi - run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && TRIGGERS+=("load=${LOAD}") -fi - -# Zombie processes -if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then - info "$ICON_GEAR Checking zombie processes..." - ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" || echo 0) - TRIGGERED=false - if [[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]]; then - error "$ICON_GEAR Zombie processes: $ZOMBIE_COUNT — threshold $SYS_WATCHDOG_ZOMBIE_LIMIT" - TRIGGERED=true - else - success "$ICON_GEAR Zombie processes: $ZOMBIE_COUNT" - fi - run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && TRIGGERS+=("zombies=${ZOMBIE_COUNT}") -fi - -# Docker daemon health -if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then - info "$ICON_CONTAINERS Checking Docker daemon..." - TRIGGERED=false - if ! timeout 10 docker ps >/dev/null 2>&1; then - error "$ICON_CONTAINERS Docker daemon is not responding" - TRIGGERED=true - else - success "$ICON_CONTAINERS Docker daemon is healthy" - fi - run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon unresponsive" && TRIGGERS+=("docker_daemon") -fi - -# Required containers from skip list -if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then - info "$ICON_CONTAINERS Checking persistent failed containers..." - FAILED_CONTAINERS=() - while IFS= read -r container; do - [[ -z "$container" ]] && continue - STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") - if [[ "$STATUS" != "true" ]]; then - error "$ICON_NOT_RUNNING $container still not running (from skip list)" - FAILED_CONTAINERS+=("$container") - fi - done < "$SYS_WATCHDOG_FAILED_FILE" - - if [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]]; then - TRIGGERED=true - run_strike_check "failed_containers" "$TRIGGERED" "required containers stopped" && TRIGGERS+=("containers=${FAILED_CONTAINERS[*]}") - fi -fi - -# ----------------------------------------------------------------------------------------------- -# No triggers — exit cleanly -# ----------------------------------------------------------------------------------------------- -if [[ ${#TRIGGERS[@]} -eq 0 ]]; then - echo "" - success "No reboot conditions met — system is healthy" - exit 0 -fi - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_REBOOT_SMART Reboot Triggered ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_REBOOT_SMART Reboot Triggered ━━━" -for t in "${TRIGGERS[@]}"; do - echo " $ICON_REBOOT_SMART $t" -done -echo "" - -# Check abort conditions before proceeding -if ! check_abort_conditions; then - exit 0 -fi - -# Reboot loop protection — check recent reboot count -RECENT_REBOOTS=$(count_recent_reboots) -info "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT" - -if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then - error "Reboot limit hit — $RECENT_REBOOTS reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs — shutting down instead" - notify "Reboot loop detected on $(hostname) — shutting down after $RECENT_REBOOTS reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs — conditions: ${TRIGGERS[*]}" "System Watchdog" "warning" - - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would shutdown now" - exit 0 - fi - - sync - /sbin/poweroff - exit 0 -fi - -notify "System watchdog reboot triggered on $(hostname) — conditions: ${TRIGGERS[*]}" "System Watchdog" "warning" - -if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — reboot sequence would begin now" - echo "" - echo "━━━━━ $ICON_SUMMARY SYSTEM WATCHDOG SUMMARY ━━━━━" - echo "$ICON_REBOOT_SMART Triggers: ${TRIGGERS[*]}" - echo "$ICON_REBOOT_SMART Recent reboots: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT" - echo "$ICON_WARN Status: DRY RUN — no reboot executed" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - exit 0 -fi - -# ----------------------------------------------------------------------------------------------- -# Graceful shutdown sequence -# ----------------------------------------------------------------------------------------------- -info "Logging reboot timestamp..." -log_reboot - -info "Shutting down VMs..." -if command -v virsh >/dev/null 2>&1; then - for VM in $(virsh list --name 2>/dev/null); do - [[ -z "$VM" ]] && continue - info "Shutting down VM: $VM" - virsh shutdown "$VM" >/dev/null 2>&1 - done - info "Waiting 30s for VMs..." - sleep 30 -fi - -info "Stopping Docker containers..." -if command -v docker >/dev/null 2>&1; then - docker ps -q | xargs -r docker stop >/dev/null 2>&1 - success "Docker containers stopped" -fi - -info "Stopping User Scripts..." -pkill -f "/tmp/user.scripts" 2>/dev/null || true - -info "Syncing disks..." -sync -success "Disks synced" - -echo "" -echo "$ICON_REBOOT_SMART Rebooting system NOW..." -sleep 5 -/sbin/reboot \ No newline at end of file +done \ No newline at end of file