Add verbose log() coverage across all watchdog scripts
docker_watchdog: config dump at startup (thresholds/limits), skip list shown when active, per-container healthy log for Tier1 required + mem/CPU monitored containers stability_watchdog: config dump with all tier thresholds, log() on pass for rootfs/log/tmp/load/zombies/NIC checks (previously silent on clean) resource_watchdog: config dump with all pressure thresholds and container lists, log normal pressure state with live RAM/load values system_watchdog: per-script timing on each child script run network_watchdog: config dump (internet URL, DDNS domain/container, NPM URL, strike limit) storage_watchdog: config dump (growth threshold, log max, paths, suppress ceilings) webgui_watchdog: log nginx worker and php-fpm worker counts on healthy check
This commit is contained in:
Gmer4Lfe
@@ -124,6 +124,9 @@ detect_hosts
|
||||
[[ "${NETWORK_WATCHDOG_ENABLED:-true}" != "true" ]] && echo "Network watchdog disabled" && exit 0
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||
|
||||
log "$ICON_GEAR Config: internet=${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1} timeout=${NETWORK_WATCHDOG_INTERNET_TIMEOUT:-5}s tailscale=${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true} npm-strikes=${NETWORK_WATCHDOG_NPM_STRIKE_LIMIT:-2}"
|
||||
log "$ICON_NET DDNS: ${DDNS_DOMAIN:-not configured} → ${DDNS_CONTAINER:-no container} NPM: ${NPM_URL:-not configured}"
|
||||
|
||||
touch "${NETWORK_WATCHDOG_NPM_STATE_FILE}" 2>/dev/null
|
||||
|
||||
# ━━━ Strike helpers ━━━
|
||||
|
||||
@@ -157,6 +157,10 @@ detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be truncated"
|
||||
|
||||
log "$ICON_GEAR Config: growth-thresh=${WATCHDOG_APPDATA_GROWTH_GB:-2}GB/cycle log-max=${WATCHDOG_APPDATA_LOG_MAX_GB:-2}GB truncate=${WATCHDOG_APPDATA_TRUNCATE_LOGS:-false} strikes=${WATCHDOG_APPDATA_STRIKE_LIMIT:-3}"
|
||||
log "$ICON_DISK Paths: ${WATCHDOG_APPDATA_PATHS[*]:-none configured}"
|
||||
[[ ${#WATCHDOG_APPDATA_SIZES[@]} -gt 0 ]] && log "$ICON_GEAR Suppress ceilings: $(for k in "${!WATCHDOG_APPDATA_SIZES[@]}"; do printf '%s=%sMB ' "$k" "${WATCHDOG_APPDATA_SIZES[$k]}"; done)"
|
||||
|
||||
touch "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null
|
||||
touch "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null
|
||||
|
||||
|
||||
@@ -163,6 +163,9 @@ log "WebGUI check — $WEBGUI_URL"
|
||||
|
||||
# ── Healthy — completely silent ───────────────────────────────────────────────────────────────
|
||||
if check_webgui; then
|
||||
_nginx_count=$(pgrep -cx nginx 2>/dev/null || echo 0)
|
||||
_fpm_count=$(pgrep -fc "php-fpm" 2>/dev/null || echo 0)
|
||||
log "$ICON_WEBGUI WebGUI responding ✅ — nginx workers:${_nginx_count} php-fpm workers:${_fpm_count}"
|
||||
echo "WebGUI responding — healthy ✅"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -246,6 +246,9 @@ fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||
|
||||
log "$ICON_GEAR Config: grace=${WATCHDOG_STARTUP_GRACE}s mem-soft=${SOFT_MEM_THRESHOLD}% cpu-soft=${SOFT_CPU_THRESHOLD}% cpu-hard=${HARD_CPU_THRESHOLD}% cpu-limit=${CPU_FAIL_LIMIT} http-limit=${RESP_FAIL_LIMIT} daemon-timeout=${DOCKER_TIMEOUT}s"
|
||||
log "$ICON_CONTAINERS Tier1: watched=${#WATCHDOG_CONTAINERS[@]} required=${#WATCHDOG_REQUIRED_CONTAINERS[@]} urls=${#WATCHDOG_CONTAINER_URLS[@]} restart-limit=${WATCHDOG_CONTAINER_RESTART_LIMIT}/${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
|
||||
|
||||
# Validate unRAID-specific commands used by this script
|
||||
# If rc.docker is missing or changed, daemon restart will fail — better to know now
|
||||
validate_unraid_cmd "/etc/rc.d/rc.docker" "" "" "Docker rc.d script" || warn "rc.docker not found — daemon restart unavailable if needed"
|
||||
@@ -623,6 +626,11 @@ CYCLE_START=$(date +%s)
|
||||
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
|
||||
done
|
||||
|
||||
# ── Skip list visibility ─────────────────────────────────────────────────────────────────
|
||||
local _skip_contents
|
||||
_skip_contents=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
|
||||
[[ -n "$_skip_contents" ]] && warn "$ICON_SKIP Skip list active: $_skip_contents — manual intervention needed"
|
||||
|
||||
# ── Docker daemon health check — first check every run ──────────────────────────────────
|
||||
# If daemon is hung all container operations will fail — check first, skip run if down
|
||||
if ! check_docker_daemon; then
|
||||
@@ -681,6 +689,7 @@ CYCLE_START=$(date +%s)
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
# Running — clear any strikes
|
||||
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
log "$ICON_RUNNING $container — running ✅"
|
||||
else
|
||||
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
|
||||
STRIKES=$(( STRIKES + 1 ))
|
||||
@@ -765,6 +774,7 @@ CYCLE_START=$(date +%s)
|
||||
else
|
||||
# Normal — clear CPU strikes
|
||||
set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
|
||||
log "$container — CPU ${CPU_NORM}% | MEM ${MEM_MB}MB / ${MEM_LIMIT_MB}MB ✅"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
@@ -133,6 +133,9 @@ detect_hosts
|
||||
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
log "$ICON_GEAR Config: soft=RAM<${RW_RAM_SOFT_GB}GB/load≥${RW_LOAD_SOFT_THRESH} medium=RAM<${RW_RAM_MEDIUM_GB}GB/load≥${RW_LOAD_MEDIUM_THRESH} hard=RAM<${RW_RAM_HARD_GB}GB recover=RAM≥${RW_RAM_RECOVER_GB}GB cycles=${RW_RECOVER_CYCLES}"
|
||||
log "$ICON_CONTAINERS Pause at medium: ${RW_PAUSE_CONTAINERS[*]:-none} Stop at hard: ${RW_STOP_CONTAINERS[*]:-none}"
|
||||
|
||||
touch "$RW_STATE_FILE" 2>/dev/null || {
|
||||
error "Cannot create state file: $RW_STATE_FILE"
|
||||
exit 1
|
||||
@@ -570,7 +573,7 @@ else
|
||||
if [[ "$CURRENT_LEVEL" -gt 0 ]]; then
|
||||
echo "Pressure holding at level $CURRENT_LEVEL — waiting for sustained recovery"
|
||||
else
|
||||
echo "System at normal pressure ✅"
|
||||
log "System normal — RAM ${MEM_GB}GB free | load ${LOAD} | ${TOTAL_CORES} cores | SABnzbd=${RW_SABNZBD_ENABLED:-true} qBit=${RW_QBIT_ENABLED:-true} ✅"
|
||||
fi
|
||||
rm_state_set "rm_recover_cycles" 0
|
||||
fi
|
||||
|
||||
@@ -142,6 +142,9 @@ done
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboots or container shutdowns will occur"
|
||||
|
||||
log "$ICON_GEAR Config: strikes=${SYS_WATCHDOG_STRIKE_LIMIT} reboot-limit=${SYS_WATCHDOG_REBOOT_LIMIT}/${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr oom-limit=${SYS_WATCHDOG_OOM_LIMIT}"
|
||||
log "$ICON_GEAR Tiers: rootfs-crit=${SYS_WATCHDOG_ROOTFS_CRITICAL_PCT}% rootfs-warn=${SYS_WATCHDOG_ROOTFS_PCT}% ram-reboot=${SYS_WATCHDOG_MEM_GB}GB load=${SYS_WATCHDOG_LOAD_MULTIPLIER}x(${TOTAL_CORES}cores) cpu-temp=${SYS_WATCHDOG_CPU_TEMP_MAX}°C zombies=${SYS_WATCHDOG_ZOMBIE_LIMIT}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
@@ -570,6 +573,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
||||
[[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true
|
||||
run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \
|
||||
TRIGGERS+=("rootfs=${ROOTFS_USED}%")
|
||||
[[ "$TRIGGERED" == false ]] && log "rootfs ${ROOTFS_USED}% ✅ (warn at ${SYS_WATCHDOG_ROOTFS_PCT}%)"
|
||||
fi
|
||||
|
||||
# ── /var/log ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -579,6 +583,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
||||
[[ "${LOG_USED:-0}" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true
|
||||
run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \
|
||||
TRIGGERS+=("log=${LOG_USED}%")
|
||||
[[ "$TRIGGERED" == false ]] && log "/var/log ${LOG_USED:-?}% ✅ (warn at ${SYS_WATCHDOG_LOG_PCT}%)"
|
||||
fi
|
||||
|
||||
# ── /tmp ─────────────────────────────────────────────────────────────────────────────────
|
||||
@@ -672,6 +677,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
||||
[[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true
|
||||
run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \
|
||||
TRIGGERS+=("load=${LOAD}")
|
||||
[[ "$TRIGGERED" == false ]] && log "load avg ${LOAD} ✅ (warn at ${LOAD_THRESHOLD} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}×${TOTAL_CORES} cores)"
|
||||
fi
|
||||
|
||||
# ── Zombie processes ─────────────────────────────────────────────────────────────────────
|
||||
@@ -682,6 +688,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
||||
[[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true
|
||||
run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \
|
||||
TRIGGERS+=("zombies=${ZOMBIE_COUNT}")
|
||||
[[ "$TRIGGERED" == false ]] && log "zombies ${ZOMBIE_COUNT} ✅ (warn at ${SYS_WATCHDOG_ZOMBIE_LIMIT})"
|
||||
fi
|
||||
|
||||
# ── Array disk errors — accumulating mdstat errors ────────────────────────────────────────
|
||||
@@ -709,10 +716,12 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
||||
if [[ "$SYS_WATCHDOG_CHECK_NETWORK" == true ]]; then
|
||||
NIC="${SYS_WATCHDOG_NIC:-eth0}"
|
||||
NIC_STATE=$(cat "/sys/class/net/${NIC}/operstate" 2>/dev/null || echo "unknown")
|
||||
NIC_SPEED=$(cat "/sys/class/net/${NIC}/speed" 2>/dev/null || echo "?")
|
||||
TRIGGERED=false
|
||||
[[ "$NIC_STATE" != "up" ]] && TRIGGERED=true
|
||||
run_strike_check "network" "$TRIGGERED" "${NIC} state: ${NIC_STATE}" && \
|
||||
TRIGGERS+=("nic_down=${NIC}")
|
||||
[[ "$TRIGGERED" == false ]] && log "$NIC ${NIC_STATE} @ ${NIC_SPEED}Mbps ✅"
|
||||
fi
|
||||
|
||||
# ── sshd — try restart before escalating ─────────────────────────────────────────────────
|
||||
|
||||
@@ -119,11 +119,12 @@ for entry in "${SYSTEM_WATCHDOG_SCRIPTS[@]}"; do
|
||||
continue
|
||||
fi
|
||||
|
||||
_ss=$(date +%s)
|
||||
if bash "$script_path"; then
|
||||
log "$script_name — done ✅"
|
||||
log "$script_name — done in $(format_duration $(( $(date +%s) - _ss ))) ✅"
|
||||
PASSED+=("$script_name")
|
||||
else
|
||||
warn "$script_name — exit non-zero (issues found or fixed) — continuing"
|
||||
warn "$script_name — exit non-zero in $(format_duration $(( $(date +%s) - _ss ))) (issues found or fixed) — continuing"
|
||||
FAILED+=("$script_name")
|
||||
fi
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user