Add verbose log() coverage across all watchdog scripts

docker_watchdog: config dump at startup (thresholds/limits), skip list shown when active, per-container healthy log for Tier1 required + mem/CPU monitored containers
stability_watchdog: config dump with all tier thresholds, log() on pass for rootfs/log/tmp/load/zombies/NIC checks (previously silent on clean)
resource_watchdog: config dump with all pressure thresholds and container lists, log normal pressure state with live RAM/load values
system_watchdog: per-script timing on each child script run
network_watchdog: config dump (internet URL, DDNS domain/container, NPM URL, strike limit)
storage_watchdog: config dump (growth threshold, log max, paths, suppress ceilings)
webgui_watchdog: log nginx worker and php-fpm worker counts on healthy check
This commit is contained in:
Gmer4Lfe
2026-06-01 20:56:46 -04:00
parent bc70ebe5ee
commit 08fc551d9f
7 changed files with 36 additions and 3 deletions
+9
View File
@@ -142,6 +142,9 @@ done
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboots or container shutdowns will occur"
log "$ICON_GEAR Config: strikes=${SYS_WATCHDOG_STRIKE_LIMIT} reboot-limit=${SYS_WATCHDOG_REBOOT_LIMIT}/${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr oom-limit=${SYS_WATCHDOG_OOM_LIMIT}"
log "$ICON_GEAR Tiers: rootfs-crit=${SYS_WATCHDOG_ROOTFS_CRITICAL_PCT}% rootfs-warn=${SYS_WATCHDOG_ROOTFS_PCT}% ram-reboot=${SYS_WATCHDOG_MEM_GB}GB load=${SYS_WATCHDOG_LOAD_MULTIPLIER}x(${TOTAL_CORES}cores) cpu-temp=${SYS_WATCHDOG_CPU_TEMP_MAX}°C zombies=${SYS_WATCHDOG_ZOMBIE_LIMIT}"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
@@ -570,6 +573,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
[[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true
run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \
TRIGGERS+=("rootfs=${ROOTFS_USED}%")
[[ "$TRIGGERED" == false ]] && log "rootfs ${ROOTFS_USED}% ✅ (warn at ${SYS_WATCHDOG_ROOTFS_PCT}%)"
fi
# ── /var/log ─────────────────────────────────────────────────────────────────────────────
@@ -579,6 +583,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
[[ "${LOG_USED:-0}" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true
run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \
TRIGGERS+=("log=${LOG_USED}%")
[[ "$TRIGGERED" == false ]] && log "/var/log ${LOG_USED:-?}% ✅ (warn at ${SYS_WATCHDOG_LOG_PCT}%)"
fi
# ── /tmp ─────────────────────────────────────────────────────────────────────────────────
@@ -672,6 +677,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
[[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true
run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \
TRIGGERS+=("load=${LOAD}")
[[ "$TRIGGERED" == false ]] && log "load avg ${LOAD} ✅ (warn at ${LOAD_THRESHOLD} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}×${TOTAL_CORES} cores)"
fi
# ── Zombie processes ─────────────────────────────────────────────────────────────────────
@@ -682,6 +688,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
[[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true
run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \
TRIGGERS+=("zombies=${ZOMBIE_COUNT}")
[[ "$TRIGGERED" == false ]] && log "zombies ${ZOMBIE_COUNT} ✅ (warn at ${SYS_WATCHDOG_ZOMBIE_LIMIT})"
fi
# ── Array disk errors — accumulating mdstat errors ────────────────────────────────────────
@@ -709,10 +716,12 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
if [[ "$SYS_WATCHDOG_CHECK_NETWORK" == true ]]; then
NIC="${SYS_WATCHDOG_NIC:-eth0}"
NIC_STATE=$(cat "/sys/class/net/${NIC}/operstate" 2>/dev/null || echo "unknown")
NIC_SPEED=$(cat "/sys/class/net/${NIC}/speed" 2>/dev/null || echo "?")
TRIGGERED=false
[[ "$NIC_STATE" != "up" ]] && TRIGGERED=true
run_strike_check "network" "$TRIGGERED" "${NIC} state: ${NIC_STATE}" && \
TRIGGERS+=("nic_down=${NIC}")
[[ "$TRIGGERED" == false ]] && log "$NIC ${NIC_STATE} @ ${NIC_SPEED}Mbps ✅"
fi
# ── sshd — try restart before escalating ─────────────────────────────────────────────────