diff --git a/Watchdogs/System/network_watchdog.sh b/Watchdogs/System/network_watchdog.sh index 621c1c3..74461c6 100755 --- a/Watchdogs/System/network_watchdog.sh +++ b/Watchdogs/System/network_watchdog.sh @@ -124,6 +124,9 @@ detect_hosts [[ "${NETWORK_WATCHDOG_ENABLED:-true}" != "true" ]] && echo "Network watchdog disabled" && exit 0 [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" +log "$ICON_GEAR Config: internet=${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1} timeout=${NETWORK_WATCHDOG_INTERNET_TIMEOUT:-5}s tailscale=${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true} npm-strikes=${NETWORK_WATCHDOG_NPM_STRIKE_LIMIT:-2}" +log "$ICON_NET DDNS: ${DDNS_DOMAIN:-not configured} → ${DDNS_CONTAINER:-no container} NPM: ${NPM_URL:-not configured}" + touch "${NETWORK_WATCHDOG_NPM_STATE_FILE}" 2>/dev/null # ━━━ Strike helpers ━━━ diff --git a/Watchdogs/System/storage_watchdog.sh b/Watchdogs/System/storage_watchdog.sh index f840af1..cf9cdd2 100755 --- a/Watchdogs/System/storage_watchdog.sh +++ b/Watchdogs/System/storage_watchdog.sh @@ -157,6 +157,10 @@ detect_hosts [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be truncated" +log "$ICON_GEAR Config: growth-thresh=${WATCHDOG_APPDATA_GROWTH_GB:-2}GB/cycle log-max=${WATCHDOG_APPDATA_LOG_MAX_GB:-2}GB truncate=${WATCHDOG_APPDATA_TRUNCATE_LOGS:-false} strikes=${WATCHDOG_APPDATA_STRIKE_LIMIT:-3}" +log "$ICON_DISK Paths: ${WATCHDOG_APPDATA_PATHS[*]:-none configured}" +[[ ${#WATCHDOG_APPDATA_SIZES[@]} -gt 0 ]] && log "$ICON_GEAR Suppress ceilings: $(for k in "${!WATCHDOG_APPDATA_SIZES[@]}"; do printf '%s=%sMB ' "$k" "${WATCHDOG_APPDATA_SIZES[$k]}"; done)" + touch "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null touch "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null diff --git a/Watchdogs/System/webgui_watchdog.sh b/Watchdogs/System/webgui_watchdog.sh index a19dc06..f970467 100755 --- a/Watchdogs/System/webgui_watchdog.sh +++ b/Watchdogs/System/webgui_watchdog.sh @@ -163,6 +163,9 @@ log "WebGUI check — $WEBGUI_URL" # ── Healthy — completely silent ─────────────────────────────────────────────────────────────── if check_webgui; then + _nginx_count=$(pgrep -cx nginx 2>/dev/null || echo 0) + _fpm_count=$(pgrep -fc "php-fpm" 2>/dev/null || echo 0) + log "$ICON_WEBGUI WebGUI responding ✅ — nginx workers:${_nginx_count} php-fpm workers:${_fpm_count}" echo "WebGUI responding — healthy ✅" exit 0 fi diff --git a/Watchdogs/docker_watchdog.sh b/Watchdogs/docker_watchdog.sh index 741fe37..586d7c0 100755 --- a/Watchdogs/docker_watchdog.sh +++ b/Watchdogs/docker_watchdog.sh @@ -246,6 +246,9 @@ fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" +log "$ICON_GEAR Config: grace=${WATCHDOG_STARTUP_GRACE}s mem-soft=${SOFT_MEM_THRESHOLD}% cpu-soft=${SOFT_CPU_THRESHOLD}% cpu-hard=${HARD_CPU_THRESHOLD}% cpu-limit=${CPU_FAIL_LIMIT} http-limit=${RESP_FAIL_LIMIT} daemon-timeout=${DOCKER_TIMEOUT}s" +log "$ICON_CONTAINERS Tier1: watched=${#WATCHDOG_CONTAINERS[@]} required=${#WATCHDOG_REQUIRED_CONTAINERS[@]} urls=${#WATCHDOG_CONTAINER_URLS[@]} restart-limit=${WATCHDOG_CONTAINER_RESTART_LIMIT}/${WATCHDOG_CONTAINER_RESTART_WINDOW}h" + # Validate unRAID-specific commands used by this script # If rc.docker is missing or changed, daemon restart will fail — better to know now validate_unraid_cmd "/etc/rc.d/rc.docker" "" "" "Docker rc.d script" || warn "rc.docker not found — daemon restart unavailable if needed" @@ -623,6 +626,11 @@ CYCLE_START=$(date +%s) [[ -n "$c" ]] && IGNORE_MAP["$c"]=1 done + # ── Skip list visibility ───────────────────────────────────────────────────────────────── + local _skip_contents + _skip_contents=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null | tr '\n' ' ' | xargs) + [[ -n "$_skip_contents" ]] && warn "$ICON_SKIP Skip list active: $_skip_contents — manual intervention needed" + # ── Docker daemon health check — first check every run ────────────────────────────────── # If daemon is hung all container operations will fail — check first, skip run if down if ! check_docker_daemon; then @@ -681,6 +689,7 @@ CYCLE_START=$(date +%s) if [[ "$STATUS" == "true" ]]; then # Running — clear any strikes set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" + log "$ICON_RUNNING $container — running ✅" else STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE") STRIKES=$(( STRIKES + 1 )) @@ -765,6 +774,7 @@ CYCLE_START=$(date +%s) else # Normal — clear CPU strikes set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE" + log "$container — CPU ${CPU_NORM}% | MEM ${MEM_MB}MB / ${MEM_LIMIT_MB}MB ✅" fi done fi diff --git a/Watchdogs/resource_watchdog.sh b/Watchdogs/resource_watchdog.sh index aba89cd..aac51f9 100755 --- a/Watchdogs/resource_watchdog.sh +++ b/Watchdogs/resource_watchdog.sh @@ -133,6 +133,9 @@ detect_hosts DOCKER_TIMEOUT=15 +log "$ICON_GEAR Config: soft=RAM<${RW_RAM_SOFT_GB}GB/load≥${RW_LOAD_SOFT_THRESH} medium=RAM<${RW_RAM_MEDIUM_GB}GB/load≥${RW_LOAD_MEDIUM_THRESH} hard=RAM<${RW_RAM_HARD_GB}GB recover=RAM≥${RW_RAM_RECOVER_GB}GB cycles=${RW_RECOVER_CYCLES}" +log "$ICON_CONTAINERS Pause at medium: ${RW_PAUSE_CONTAINERS[*]:-none} Stop at hard: ${RW_STOP_CONTAINERS[*]:-none}" + touch "$RW_STATE_FILE" 2>/dev/null || { error "Cannot create state file: $RW_STATE_FILE" exit 1 @@ -570,7 +573,7 @@ else if [[ "$CURRENT_LEVEL" -gt 0 ]]; then echo "Pressure holding at level $CURRENT_LEVEL — waiting for sustained recovery" else - echo "System at normal pressure ✅" + log "System normal — RAM ${MEM_GB}GB free | load ${LOAD} | ${TOTAL_CORES} cores | SABnzbd=${RW_SABNZBD_ENABLED:-true} qBit=${RW_QBIT_ENABLED:-true} ✅" fi rm_state_set "rm_recover_cycles" 0 fi diff --git a/Watchdogs/stability_watchdog.sh b/Watchdogs/stability_watchdog.sh index 9bbcd50..cf2cdd1 100755 --- a/Watchdogs/stability_watchdog.sh +++ b/Watchdogs/stability_watchdog.sh @@ -142,6 +142,9 @@ done [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboots or container shutdowns will occur" +log "$ICON_GEAR Config: strikes=${SYS_WATCHDOG_STRIKE_LIMIT} reboot-limit=${SYS_WATCHDOG_REBOOT_LIMIT}/${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr oom-limit=${SYS_WATCHDOG_OOM_LIMIT}" +log "$ICON_GEAR Tiers: rootfs-crit=${SYS_WATCHDOG_ROOTFS_CRITICAL_PCT}% rootfs-warn=${SYS_WATCHDOG_ROOTFS_PCT}% ram-reboot=${SYS_WATCHDOG_MEM_GB}GB load=${SYS_WATCHDOG_LOAD_MULTIPLIER}x(${TOTAL_CORES}cores) cpu-temp=${SYS_WATCHDOG_CPU_TEMP_MAX}°C zombies=${SYS_WATCHDOG_ZOMBIE_LIMIT}" + # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== @@ -570,6 +573,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S') [[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \ TRIGGERS+=("rootfs=${ROOTFS_USED}%") + [[ "$TRIGGERED" == false ]] && log "rootfs ${ROOTFS_USED}% ✅ (warn at ${SYS_WATCHDOG_ROOTFS_PCT}%)" fi # ── /var/log ───────────────────────────────────────────────────────────────────────────── @@ -579,6 +583,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S') [[ "${LOG_USED:-0}" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \ TRIGGERS+=("log=${LOG_USED}%") + [[ "$TRIGGERED" == false ]] && log "/var/log ${LOG_USED:-?}% ✅ (warn at ${SYS_WATCHDOG_LOG_PCT}%)" fi # ── /tmp ───────────────────────────────────────────────────────────────────────────────── @@ -672,6 +677,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S') [[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \ TRIGGERS+=("load=${LOAD}") + [[ "$TRIGGERED" == false ]] && log "load avg ${LOAD} ✅ (warn at ${LOAD_THRESHOLD} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}×${TOTAL_CORES} cores)" fi # ── Zombie processes ───────────────────────────────────────────────────────────────────── @@ -682,6 +688,7 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S') [[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \ TRIGGERS+=("zombies=${ZOMBIE_COUNT}") + [[ "$TRIGGERED" == false ]] && log "zombies ${ZOMBIE_COUNT} ✅ (warn at ${SYS_WATCHDOG_ZOMBIE_LIMIT})" fi # ── Array disk errors — accumulating mdstat errors ──────────────────────────────────────── @@ -709,10 +716,12 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S') if [[ "$SYS_WATCHDOG_CHECK_NETWORK" == true ]]; then NIC="${SYS_WATCHDOG_NIC:-eth0}" NIC_STATE=$(cat "/sys/class/net/${NIC}/operstate" 2>/dev/null || echo "unknown") + NIC_SPEED=$(cat "/sys/class/net/${NIC}/speed" 2>/dev/null || echo "?") TRIGGERED=false [[ "$NIC_STATE" != "up" ]] && TRIGGERED=true run_strike_check "network" "$TRIGGERED" "${NIC} state: ${NIC_STATE}" && \ TRIGGERS+=("nic_down=${NIC}") + [[ "$TRIGGERED" == false ]] && log "$NIC ${NIC_STATE} @ ${NIC_SPEED}Mbps ✅" fi # ── sshd — try restart before escalating ───────────────────────────────────────────────── diff --git a/Watchdogs/system_watchdog.sh b/Watchdogs/system_watchdog.sh index c07ac8b..d1fa60f 100755 --- a/Watchdogs/system_watchdog.sh +++ b/Watchdogs/system_watchdog.sh @@ -119,11 +119,12 @@ for entry in "${SYSTEM_WATCHDOG_SCRIPTS[@]}"; do continue fi + _ss=$(date +%s) if bash "$script_path"; then - log "$script_name — done ✅" + log "$script_name — done in $(format_duration $(( $(date +%s) - _ss ))) ✅" PASSED+=("$script_name") else - warn "$script_name — exit non-zero (issues found or fixed) — continuing" + warn "$script_name — exit non-zero in $(format_duration $(( $(date +%s) - _ss ))) (issues found or fixed) — continuing" FAILED+=("$script_name") fi done