Gmer4Lfe
c377ddfcca
Finishes the pass: every script now documents its safeguards, and the deliberate absences in the sourced libraries are recorded so they are not "corrected" later.
328 lines
14 KiB
Bash
Executable File
328 lines
14 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= WebGUI Watchdog ============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Monitors the unRAID WebGUI and restarts services if unresponsive. Uses a
|
|
# three-step escalating strategy — lightest fix first, heaviest last. Called
|
|
# by system_watchdog.sh each cycle. Silent when healthy.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# Escalation Path
|
|
# WebGUI responding → log() + exit 0 (completely silent ✅)
|
|
#
|
|
# Not responding:
|
|
# Step 1 — nginx restart
|
|
# Lightest fix — handles most transient WebGUI failures:
|
|
# nginx crash, worker stuck, connection timeout.
|
|
# Wait WEBGUI_NGINX_WAIT seconds → recheck.
|
|
#
|
|
# Step 2 — php-fpm restart
|
|
# WebGUI runs through PHP-FPM. Worker exhaustion causes silent
|
|
# failure — requests queue and the WebGUI appears frozen.
|
|
# Wait WEBGUI_PHP_WAIT seconds → recheck.
|
|
#
|
|
# Step 3 — emhttp restart
|
|
# Heaviest fix. emhttp is the unRAID management daemon.
|
|
# Array, Docker, and shares stay running — only WebGUI
|
|
# management restarts. Takes longer — WEBGUI_EMHTTP_WAIT.
|
|
# Wait WEBGUI_EMHTTP_WAIT seconds → recheck.
|
|
#
|
|
# All three failed → notify, manual intervention needed → exit 1.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Lightest Fix First
|
|
# Escalation runs cheapest-to-heaviest, and stops the moment the WebGUI answers. Most
|
|
# hangs clear with a php-fpm restart; going straight to the heavy remedy would take the
|
|
# whole management interface down for a fault a service reload would have fixed.
|
|
#
|
|
# The WebGUI Is Not the Server
|
|
# An unresponsive WebGUI does not mean an unhealthy machine. Docker, the array and every
|
|
# share keep working while emhttp is wedged, so nothing here reboots or touches storage —
|
|
# the remedy stays scoped to the web stack.
|
|
#
|
|
# Verify After Every Step
|
|
# Responsiveness is re-tested between escalation steps rather than assuming a restart
|
|
# worked. Without that the script would walk the full ladder every time, restarting
|
|
# services that were already fixed one step earlier.
|
|
#
|
|
# Silent When Healthy
|
|
# Runs every cycle via system_watchdog.sh and prints nothing on a working WebGUI. A
|
|
# per-minute "all good" line would bury the one cycle that mattered.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root Required
|
|
# Service restart commands require root.
|
|
#
|
|
# Single Instance Lock
|
|
# acquire_lock prevents concurrent runs double-restarting services.
|
|
#
|
|
# Process Verify After Each Restart
|
|
# pgrep check after each rc.* command — errors if process not running.
|
|
#
|
|
# Silent When Healthy
|
|
# Completely silent on healthy cycles. Only produces output when recovering.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# WEBGUI_URL
|
|
# URL to check for WebGUI response. (default: http://localhost)
|
|
#
|
|
# WEBGUI_TIMEOUT
|
|
# curl timeout in seconds. (default: 5)
|
|
#
|
|
# WEBGUI_NGINX_WAIT
|
|
# Seconds after nginx restart before rechecking. (default: 15)
|
|
#
|
|
# WEBGUI_PHP_WAIT
|
|
# Seconds after php-fpm restart before rechecking. (default: 10)
|
|
#
|
|
# WEBGUI_EMHTTP_WAIT
|
|
# Seconds after emhttp restart before rechecking. (default: 30)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# webgui_watchdog.sh
|
|
# Check WebGUI. Escalate through nginx → php-fpm → emhttp if unresponsive.
|
|
#
|
|
# webgui_watchdog.sh --dry-run
|
|
# Show which services would be restarted. No restarts, no waits.
|
|
#
|
|
# webgui_watchdog.sh --status
|
|
# Show current WebGUI response state and nginx/php-fpm/emhttp process states.
|
|
#
|
|
# webgui_watchdog.sh --log
|
|
# Verbose output — show each check, each restart attempt, each wait.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../../../../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
platform_require_cmd \
|
|
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
|
"" "" \
|
|
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
|
|
|
acquire_lock
|
|
|
|
detect_hosts
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
|
|
echo "$ICON_WEBGUI Timeouts: curl=${WEBGUI_TIMEOUT}s nginx=${WEBGUI_NGINX_WAIT}s php=${WEBGUI_PHP_WAIT}s emhttp=${WEBGUI_EMHTTP_WAIT}s"
|
|
echo ""
|
|
|
|
if curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1; then
|
|
echo " $ICON_SUCCESS WebGUI: responding ✅"
|
|
else
|
|
echo " $ICON_ERROR WebGUI: NOT responding"
|
|
fi
|
|
|
|
platform_is_service_running nginx && \
|
|
echo " $ICON_SUCCESS nginx: running ✅" || \
|
|
echo " $ICON_ERROR nginx: NOT running"
|
|
|
|
platform_is_service_running php-fpm && \
|
|
FPM_COUNT=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?") && \
|
|
echo " $ICON_SUCCESS php-fpm: running ($FPM_COUNT workers) ✅" || \
|
|
echo " $ICON_ERROR php-fpm: NOT running"
|
|
|
|
platform_is_service_running emhttp && \
|
|
echo " $ICON_SUCCESS emhttp: running ✅" || \
|
|
echo " $ICON_ERROR emhttp: NOT running"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ── CHECK AND ESCALATE ────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
check_webgui() {
|
|
curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1
|
|
}
|
|
|
|
START=$(date +%s)
|
|
RECOVERY_ACTION=""
|
|
RECOVERY_OK=false
|
|
|
|
log "WebGUI check — $WEBGUI_URL"
|
|
|
|
# ── Healthy — completely silent ───────────────────────────────────────────────────────────────
|
|
if check_webgui; then
|
|
_nginx_count=$(pgrep -cx nginx 2>/dev/null || echo 0)
|
|
_fpm_count=$(pgrep -fc "php-fpm" 2>/dev/null || echo 0)
|
|
log "$ICON_WEBGUI WebGUI responding ✅ — nginx workers:${_nginx_count} php-fpm workers:${_fpm_count}"
|
|
echo "WebGUI responding — healthy ✅"
|
|
exit 0
|
|
fi
|
|
|
|
# ── Not responding — begin escalation ────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo ""
|
|
warn "WebGUI not responding at $WEBGUI_URL — beginning escalation"
|
|
|
|
# ── Step 1 — nginx restart ────────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 1 — nginx Restart ━━━"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would restart nginx"
|
|
else
|
|
warn "Restarting nginx..."
|
|
if platform_restart_service nginx; then
|
|
# Verify nginx actually running
|
|
sleep 2
|
|
if platform_is_service_running nginx; then
|
|
warn "nginx restarted ✅"
|
|
else
|
|
error "nginx not running after restart command"
|
|
fi
|
|
else
|
|
error "nginx restart command failed"
|
|
fi
|
|
|
|
log "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..."
|
|
sleep "$WEBGUI_NGINX_WAIT"
|
|
|
|
if check_webgui; then
|
|
RECOVERY_ACTION="nginx restart"
|
|
RECOVERY_OK=true
|
|
fi
|
|
fi
|
|
|
|
# ── Step 2 — php-fpm restart ──────────────────────────────────────────────────────────────────
|
|
if [[ "$RECOVERY_OK" == false ]]; then
|
|
echo ""
|
|
echo "━━━ Step 2 — php-fpm Restart ━━━"
|
|
warn "WebGUI still not responding — restarting php-fpm"
|
|
warn "WebGUI may be frozen due to worker exhaustion (check system_tuning_monitor.sh)"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would restart php-fpm"
|
|
else
|
|
if platform_restart_service php-fpm; then
|
|
sleep 2
|
|
if platform_is_service_running php-fpm; then
|
|
warn "php-fpm restarted ✅"
|
|
else
|
|
error "php-fpm not running after restart command"
|
|
fi
|
|
else
|
|
error "php-fpm restart command failed"
|
|
fi
|
|
|
|
log "Waiting ${WEBGUI_PHP_WAIT}s for php-fpm to recover..."
|
|
sleep "$WEBGUI_PHP_WAIT"
|
|
|
|
if check_webgui; then
|
|
RECOVERY_ACTION="php-fpm restart"
|
|
RECOVERY_OK=true
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ── Step 3 — emhttp restart ───────────────────────────────────────────────────────────────────
|
|
if [[ "$RECOVERY_OK" == false ]]; then
|
|
echo ""
|
|
echo "━━━ Step 3 — emhttp Restart ━━━"
|
|
warn "WebGUI still not responding — restarting emhttp (unRAID management daemon)"
|
|
warn "Array, Docker, and shares remain running — WebGUI management will briefly restart"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would restart emhttp"
|
|
else
|
|
if /usr/local/sbin/emhttp stop >/dev/null 2>&1 && /usr/local/sbin/emhttp start >/dev/null 2>&1; then
|
|
sleep 2
|
|
if platform_is_service_running emhttp; then
|
|
warn "emhttp restarted ✅"
|
|
else
|
|
error "emhttp not running after restart command"
|
|
fi
|
|
else
|
|
error "emhttp restart command failed"
|
|
fi
|
|
|
|
log "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..."
|
|
sleep "$WEBGUI_EMHTTP_WAIT"
|
|
|
|
if check_webgui; then
|
|
RECOVERY_ACTION="emhttp restart"
|
|
RECOVERY_OK=true
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no services restarted"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "$RECOVERY_OK" == true ]]; then
|
|
warn "$ICON_SUCCESS WebGUI recovered via: $RECOVERY_ACTION"
|
|
notify "WebGUI recovered on $(hostname) ($MY_ID) via $RECOVERY_ACTION — monitor for recurrence" \
|
|
"WebGUI Watchdog" "warning"
|
|
else
|
|
echo "$ICON_ERROR Status: UNRECOVERED — all three restart steps failed"
|
|
echo "$ICON_ERROR Manual intervention needed:"
|
|
echo " 1. Check: pgrep nginx; pgrep emhttpd"
|
|
echo " 2. Check: journalctl -u nginx --since '10 minutes ago'"
|
|
echo " 3. Try: server_reboot.sh if nothing else works"
|
|
notify "WebGUI UNRECOVERED on $(hostname) ($MY_ID) — nginx + php-fpm + emhttp restart all failed — manual intervention needed" \
|
|
"WebGUI Watchdog" "warning"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ "$RECOVERY_OK" == false && "$DRY_RUN" == false ]] && exit 1
|
|
exit 0 |