Gmer4Lfe
ea9f39a803
log() is gated on ENABLE_LOGGING — silent on normal runs. Healthy confirmations (All healthy, Network healthy, Storage healthy, WebGUI healthy) were invisible, making banners appear with nothing after them. Switched all four to plain echo so they show every cycle.
302 lines
12 KiB
Bash
302 lines
12 KiB
Bash
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ================================= WebGUI Watchdog ============================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Monitors the unRAID WebGUI and restarts services if unresponsive. Uses a
|
||
# three-step escalating strategy — lightest fix first, heaviest last. Run
|
||
# every 5–10 minutes via the User Scripts plugin. Silent when healthy.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# Escalation Path
|
||
# WebGUI responding → log() + exit 0 (completely silent ✅)
|
||
#
|
||
# Not responding:
|
||
# Step 1 — nginx restart
|
||
# Lightest fix — handles most transient WebGUI failures:
|
||
# nginx crash, worker stuck, connection timeout.
|
||
# Wait WEBGUI_NGINX_WAIT seconds → recheck.
|
||
#
|
||
# Step 2 — php-fpm restart
|
||
# WebGUI runs through PHP-FPM. Worker exhaustion causes silent
|
||
# failure — requests queue and the WebGUI appears frozen.
|
||
# Wait WEBGUI_PHP_WAIT seconds → recheck.
|
||
#
|
||
# Step 3 — emhttp restart
|
||
# Heaviest fix. emhttp is the unRAID management daemon.
|
||
# Array, Docker, and shares stay running — only WebGUI
|
||
# management restarts. Takes longer — WEBGUI_EMHTTP_WAIT.
|
||
# Wait WEBGUI_EMHTTP_WAIT seconds → recheck.
|
||
#
|
||
# All three failed → notify, manual intervention needed → exit 1.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# Root Required
|
||
# Service restart commands require root.
|
||
#
|
||
# Single Instance Lock
|
||
# acquire_lock prevents concurrent runs double-restarting services.
|
||
#
|
||
# Process Verify After Each Restart
|
||
# pgrep check after each rc.* command — errors if process not running.
|
||
#
|
||
# Silent When Healthy
|
||
# Completely silent on healthy cycles. Only produces output when recovering.
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# master.conf
|
||
#
|
||
# WEBGUI_URL
|
||
# URL to check for WebGUI response. (default: http://localhost)
|
||
#
|
||
# WEBGUI_TIMEOUT
|
||
# curl timeout in seconds. (default: 5)
|
||
#
|
||
# WEBGUI_NGINX_WAIT
|
||
# Seconds after nginx restart before rechecking. (default: 15)
|
||
#
|
||
# WEBGUI_PHP_WAIT
|
||
# Seconds after php-fpm restart before rechecking. (default: 10)
|
||
#
|
||
# WEBGUI_EMHTTP_WAIT
|
||
# Seconds after emhttp restart before rechecking. (default: 30)
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# webgui_watchdog.sh
|
||
# Check WebGUI. Escalate through nginx → php-fpm → emhttp if unresponsive.
|
||
#
|
||
# webgui_watchdog.sh --dry-run
|
||
# Show which services would be restarted. No restarts, no waits.
|
||
#
|
||
# webgui_watchdog.sh --status
|
||
# Show current WebGUI response state and nginx/php-fpm/emhttp process states.
|
||
#
|
||
# webgui_watchdog.sh --log
|
||
# Verbose output — show each check, each restart attempt, each wait.
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../../load_config.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
validate_unraid_cmd \
|
||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||
"" "" \
|
||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||
|
||
acquire_lock
|
||
|
||
detect_hosts
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG STATUS ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
|
||
echo "$ICON_WEBGUI Timeouts: curl=${WEBGUI_TIMEOUT}s nginx=${WEBGUI_NGINX_WAIT}s php=${WEBGUI_PHP_WAIT:-10}s emhttp=${WEBGUI_EMHTTP_WAIT}s"
|
||
echo ""
|
||
|
||
if curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1; then
|
||
echo " $ICON_SUCCESS WebGUI: responding ✅"
|
||
else
|
||
echo " $ICON_ERROR WebGUI: NOT responding"
|
||
fi
|
||
|
||
pgrep -x nginx >/dev/null 2>&1 && \
|
||
echo " $ICON_SUCCESS nginx: running ✅" || \
|
||
echo " $ICON_ERROR nginx: NOT running"
|
||
|
||
pgrep -f "php-fpm" >/dev/null 2>&1 && \
|
||
FPM_COUNT=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?") && \
|
||
echo " $ICON_SUCCESS php-fpm: running ($FPM_COUNT workers) ✅" || \
|
||
echo " $ICON_ERROR php-fpm: NOT running"
|
||
|
||
pgrep emhttpd >/dev/null 2>&1 && \
|
||
echo " $ICON_SUCCESS emhttp: running ✅" || \
|
||
echo " $ICON_ERROR emhttp: NOT running"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ── CHECK AND ESCALATE ────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
check_webgui() {
|
||
curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1
|
||
}
|
||
|
||
START=$(date +%s)
|
||
RECOVERY_ACTION=""
|
||
RECOVERY_OK=false
|
||
|
||
log "WebGUI check — $WEBGUI_URL"
|
||
|
||
# ── Healthy — completely silent ───────────────────────────────────────────────────────────────
|
||
if check_webgui; then
|
||
echo "WebGUI responding — healthy ✅"
|
||
exit 0
|
||
fi
|
||
|
||
# ── Not responding — begin escalation ────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo ""
|
||
warn "WebGUI not responding at $WEBGUI_URL — beginning escalation"
|
||
|
||
# ── Step 1 — nginx restart ────────────────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ Step 1 — nginx Restart ━━━"
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would restart nginx"
|
||
else
|
||
warn "Restarting nginx..."
|
||
if /etc/rc.d/rc.nginx restart >/dev/null 2>&1; then
|
||
# Verify nginx actually running
|
||
sleep 2
|
||
if pgrep -x nginx >/dev/null 2>&1; then
|
||
warn "nginx restarted ✅"
|
||
else
|
||
error "nginx not running after restart command"
|
||
fi
|
||
else
|
||
error "nginx restart command failed"
|
||
fi
|
||
|
||
log "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..."
|
||
sleep "$WEBGUI_NGINX_WAIT"
|
||
|
||
if check_webgui; then
|
||
RECOVERY_ACTION="nginx restart"
|
||
RECOVERY_OK=true
|
||
fi
|
||
fi
|
||
|
||
# ── Step 2 — php-fpm restart ──────────────────────────────────────────────────────────────────
|
||
if [[ "$RECOVERY_OK" == false ]]; then
|
||
echo ""
|
||
echo "━━━ Step 2 — php-fpm Restart ━━━"
|
||
warn "WebGUI still not responding — restarting php-fpm"
|
||
warn "WebGUI may be frozen due to worker exhaustion (check system_tuning_monitor.sh)"
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would restart php-fpm"
|
||
else
|
||
if /etc/rc.d/rc.php-fpm restart >/dev/null 2>&1; then
|
||
sleep 2
|
||
if pgrep -f "php-fpm" >/dev/null 2>&1; then
|
||
warn "php-fpm restarted ✅"
|
||
else
|
||
error "php-fpm not running after restart command"
|
||
fi
|
||
else
|
||
error "php-fpm restart command failed"
|
||
fi
|
||
|
||
log "Waiting ${WEBGUI_PHP_WAIT:-10}s for php-fpm to recover..."
|
||
sleep "${WEBGUI_PHP_WAIT:-10}"
|
||
|
||
if check_webgui; then
|
||
RECOVERY_ACTION="php-fpm restart"
|
||
RECOVERY_OK=true
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
# ── Step 3 — emhttp restart ───────────────────────────────────────────────────────────────────
|
||
if [[ "$RECOVERY_OK" == false ]]; then
|
||
echo ""
|
||
echo "━━━ Step 3 — emhttp Restart ━━━"
|
||
warn "WebGUI still not responding — restarting emhttp (unRAID management daemon)"
|
||
warn "Array, Docker, and shares remain running — WebGUI management will briefly restart"
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would restart emhttp"
|
||
else
|
||
if /usr/local/sbin/emhttp stop >/dev/null 2>&1 && /usr/local/sbin/emhttp start >/dev/null 2>&1; then
|
||
sleep 2
|
||
if pgrep emhttpd >/dev/null 2>&1; then
|
||
warn "emhttp restarted ✅"
|
||
else
|
||
error "emhttp not running after restart command"
|
||
fi
|
||
else
|
||
error "emhttp restart command failed"
|
||
fi
|
||
|
||
log "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..."
|
||
sleep "$WEBGUI_EMHTTP_WAIT"
|
||
|
||
if check_webgui; then
|
||
RECOVERY_ACTION="emhttp restart"
|
||
RECOVERY_OK=true
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
END=$(date +%s)
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Summary ━━━
|
||
# ==============================================================================================
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
|
||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||
echo ""
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — no services restarted"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
if [[ "$RECOVERY_OK" == true ]]; then
|
||
warn "$ICON_SUCCESS WebGUI recovered via: $RECOVERY_ACTION"
|
||
notify "WebGUI recovered on $(hostname) ($MY_ID) via $RECOVERY_ACTION — monitor for recurrence" \
|
||
"WebGUI Watchdog" "warning"
|
||
else
|
||
echo "$ICON_ERROR Status: UNRECOVERED — all three restart steps failed"
|
||
echo "$ICON_ERROR Manual intervention needed:"
|
||
echo " 1. Check: pgrep nginx; pgrep emhttpd"
|
||
echo " 2. Check: journalctl -u nginx --since '10 minutes ago'"
|
||
echo " 3. Try: server_reboot.sh if nothing else works"
|
||
notify "WebGUI UNRECOVERED on $(hostname) ($MY_ID) — nginx + php-fpm + emhttp restart all failed — manual intervention needed" \
|
||
"WebGUI Watchdog" "warning"
|
||
fi
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
[[ "$RECOVERY_OK" == false && "$DRY_RUN" == false ]] && exit 1
|
||
exit 0 |