Add verbose log() coverage across all watchdog scripts
docker_watchdog: config dump at startup (thresholds/limits), skip list shown when active, per-container healthy log for Tier1 required + mem/CPU monitored containers stability_watchdog: config dump with all tier thresholds, log() on pass for rootfs/log/tmp/load/zombies/NIC checks (previously silent on clean) resource_watchdog: config dump with all pressure thresholds and container lists, log normal pressure state with live RAM/load values system_watchdog: per-script timing on each child script run network_watchdog: config dump (internet URL, DDNS domain/container, NPM URL, strike limit) storage_watchdog: config dump (growth threshold, log max, paths, suppress ceilings) webgui_watchdog: log nginx worker and php-fpm worker counts on healthy check
This commit is contained in:
@@ -142,6 +142,9 @@ done
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboots or container shutdowns will occur"
|
||||
|
||||
log "$ICON_GEAR Config: strikes=${SYS_WATCHDOG_STRIKE_LIMIT} reboot-limit=${SYS_WATCHDOG_REBOOT_LIMIT}/${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr oom-limit=${SYS_WATCHDOG_OOM_LIMIT}"
|
||||
log "$ICON_GEAR Tiers: rootfs-crit=${SYS_WATCHDOG_ROOTFS_CRITICAL_PCT}% rootfs-warn=${SYS_WATCHDOG_ROOTFS_PCT}% ram-reboot=${SYS_WATCHDOG_MEM_GB}GB load=${SYS_WATCHDOG_LOAD_MULTIPLIER}x(${TOTAL_CORES}cores) cpu-temp=${SYS_WATCHDOG_CPU_TEMP_MAX}°C zombies=${SYS_WATCHDOG_ZOMBIE_LIMIT}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
@@ -570,6 +573,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
||||
[[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true
|
||||
run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \
|
||||
TRIGGERS+=("rootfs=${ROOTFS_USED}%")
|
||||
[[ "$TRIGGERED" == false ]] && log "rootfs ${ROOTFS_USED}% ✅ (warn at ${SYS_WATCHDOG_ROOTFS_PCT}%)"
|
||||
fi
|
||||
|
||||
# ── /var/log ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -579,6 +583,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
||||
[[ "${LOG_USED:-0}" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true
|
||||
run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \
|
||||
TRIGGERS+=("log=${LOG_USED}%")
|
||||
[[ "$TRIGGERED" == false ]] && log "/var/log ${LOG_USED:-?}% ✅ (warn at ${SYS_WATCHDOG_LOG_PCT}%)"
|
||||
fi
|
||||
|
||||
# ── /tmp ─────────────────────────────────────────────────────────────────────────────────
|
||||
@@ -672,6 +677,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
||||
[[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true
|
||||
run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \
|
||||
TRIGGERS+=("load=${LOAD}")
|
||||
[[ "$TRIGGERED" == false ]] && log "load avg ${LOAD} ✅ (warn at ${LOAD_THRESHOLD} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}×${TOTAL_CORES} cores)"
|
||||
fi
|
||||
|
||||
# ── Zombie processes ─────────────────────────────────────────────────────────────────────
|
||||
@@ -682,6 +688,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
||||
[[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true
|
||||
run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \
|
||||
TRIGGERS+=("zombies=${ZOMBIE_COUNT}")
|
||||
[[ "$TRIGGERED" == false ]] && log "zombies ${ZOMBIE_COUNT} ✅ (warn at ${SYS_WATCHDOG_ZOMBIE_LIMIT})"
|
||||
fi
|
||||
|
||||
# ── Array disk errors — accumulating mdstat errors ────────────────────────────────────────
|
||||
@@ -709,10 +716,12 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
||||
if [[ "$SYS_WATCHDOG_CHECK_NETWORK" == true ]]; then
|
||||
NIC="${SYS_WATCHDOG_NIC:-eth0}"
|
||||
NIC_STATE=$(cat "/sys/class/net/${NIC}/operstate" 2>/dev/null || echo "unknown")
|
||||
NIC_SPEED=$(cat "/sys/class/net/${NIC}/speed" 2>/dev/null || echo "?")
|
||||
TRIGGERED=false
|
||||
[[ "$NIC_STATE" != "up" ]] && TRIGGERED=true
|
||||
run_strike_check "network" "$TRIGGERED" "${NIC} state: ${NIC_STATE}" && \
|
||||
TRIGGERS+=("nic_down=${NIC}")
|
||||
[[ "$TRIGGERED" == false ]] && log "$NIC ${NIC_STATE} @ ${NIC_SPEED}Mbps ✅"
|
||||
fi
|
||||
|
||||
# ── sshd — try restart before escalating ─────────────────────────────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user