massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
+207
-138
@@ -1,198 +1,267 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- WebGUI Watchdog --------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Monitors unRAID's WebGUI and restarts it if unresponsive.
|
||||
# Uses an escalating restart strategy — tries nginx first, then emhttp if needed.
|
||||
# emhttp is the unRAID management daemon — restarting it is more disruptive than nginx
|
||||
# but recovers cleanly. Notification sent on any restart so you know what happened.
|
||||
# ==============================================================================================
|
||||
# ================================= WebGUI Watchdog ============================================
|
||||
# ==============================================================================================
|
||||
# Monitors the unRAID WebGUI and restarts services if unresponsive.
|
||||
# Uses a three-step escalating strategy — lightest fix first, heaviest last.
|
||||
# Run every 5-10 minutes via User Scripts plugin.
|
||||
# Silent when healthy — only produces output when something needs fixing.
|
||||
#
|
||||
# Escalation path:
|
||||
# Check WebGUI → unresponsive → restart nginx → recheck
|
||||
# Still unresponsive → restart emhttp → recheck
|
||||
# Still unresponsive → notify warning, manual intervention needed
|
||||
# ── ESCALATION PATH ───────────────────────────────────────────────────────────────────────────
|
||||
# Check WebGUI → responding → log() + exit 0 (completely silent ✅)
|
||||
#
|
||||
# Run every 5-10 minutes via cron/User Scripts plugin.
|
||||
# All configuration in Master.conf under WebGUI Watchdog section.
|
||||
# Supports --dry-run to show what would be restarted without acting.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Not responding:
|
||||
# Step 1 — Restart nginx
|
||||
# Lightest fix — handles most transient WebGUI failures
|
||||
# nginx crash, worker stuck, connection timeout
|
||||
# Wait WEBGUI_NGINX_WAIT seconds → recheck
|
||||
#
|
||||
# Step 2 — Restart php-fpm
|
||||
# WebGUI runs through PHP-FPM — worker exhaustion causes silent failure
|
||||
# php-fpm workers saturated → new requests queue → WebGUI appears frozen
|
||||
# system_tuning_monitor.sh tracks usage — this recovers it
|
||||
# Wait WEBGUI_PHP_WAIT seconds → recheck
|
||||
#
|
||||
# Step 3 — Restart emhttp
|
||||
# Heaviest fix — emhttp is the unRAID management daemon
|
||||
# Array, Docker, shares stay running — only WebGUI management restarts
|
||||
# Takes longer to recover — WEBGUI_EMHTTP_WAIT gives it time
|
||||
# Wait WEBGUI_EMHTTP_WAIT seconds → recheck
|
||||
#
|
||||
# All three failed → notify warning, manual intervention needed → exit 1
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID — used in all notifications and summary.
|
||||
# Critical on two-server setup — which server's WebGUI failed?
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents concurrent runs double-restarting services
|
||||
# detect_hosts() — MY_ID in all notifications
|
||||
# Process verify — pgrep check after each service restart
|
||||
# Silent healthy — completely silent on healthy cycle ✅
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# WEBGUI_URL — URL to check (default http://localhost)
|
||||
# WEBGUI_TIMEOUT — curl timeout in seconds (default 5)
|
||||
# WEBGUI_NGINX_WAIT — seconds after nginx restart before rechecking (default 15)
|
||||
# WEBGUI_PHP_WAIT — seconds after php-fpm restart before rechecking (default 10)
|
||||
# WEBGUI_EMHTTP_WAIT — seconds after emhttp restart before rechecking (default 30)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# webgui_restart.sh — check and recover if needed
|
||||
# webgui_restart.sh --dry-run — show what would be restarted
|
||||
# webgui_restart.sh --status — show current WebGUI and service states
|
||||
# webgui_restart.sh --log — verbose output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
|
||||
echo "$ICON_WEBGUI Curl timeout: ${WEBGUI_TIMEOUT}s"
|
||||
echo "$ICON_WEBGUI Nginx wait: ${WEBGUI_NGINX_WAIT}s"
|
||||
echo "$ICON_WEBGUI emhttp wait: ${WEBGUI_EMHTTP_WAIT}s"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
|
||||
echo "$ICON_WEBGUI Timeouts: curl=${WEBGUI_TIMEOUT}s nginx=${WEBGUI_NGINX_WAIT}s php=${WEBGUI_PHP_WAIT:-10}s emhttp=${WEBGUI_EMHTTP_WAIT}s"
|
||||
echo ""
|
||||
|
||||
# Show current state
|
||||
if curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1; then
|
||||
echo "$ICON_WEBGUI WebGUI: $ICON_RUNNING responding"
|
||||
echo " $ICON_SUCCESS WebGUI: responding ✅"
|
||||
else
|
||||
echo "$ICON_WEBGUI WebGUI: $ICON_NOT_RUNNING not responding"
|
||||
echo " $ICON_ERROR WebGUI: NOT responding"
|
||||
fi
|
||||
|
||||
pgrep -x nginx >/dev/null 2>&1 && \
|
||||
echo " $ICON_SUCCESS nginx: running ✅" || \
|
||||
echo " $ICON_ERROR nginx: NOT running"
|
||||
|
||||
pgrep -f "php-fpm" >/dev/null 2>&1 && \
|
||||
FPM_COUNT=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?") && \
|
||||
echo " $ICON_SUCCESS php-fpm: running ($FPM_COUNT workers) ✅" || \
|
||||
echo " $ICON_ERROR php-fpm: NOT running"
|
||||
|
||||
pgrep -x emhttp >/dev/null 2>&1 && \
|
||||
echo " $ICON_SUCCESS emhttp: running ✅" || \
|
||||
echo " $ICON_ERROR emhttp: NOT running"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Check if WebGUI is responding
|
||||
# ==============================================================================================
|
||||
# ── CHECK AND ESCALATE ────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
check_webgui() {
|
||||
curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# Restart nginx — lightweight fix, try first
|
||||
restart_nginx() {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart nginx"
|
||||
return 0
|
||||
fi
|
||||
|
||||
info "$ICON_WEBGUI Restarting nginx..."
|
||||
if /etc/rc.d/rc.nginx restart >/dev/null 2>&1; then
|
||||
success "nginx restarted"
|
||||
return 0
|
||||
else
|
||||
error "nginx restart failed"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Restart emhttp — heavier fix, escalate if nginx didn't help
|
||||
# emhttp drives the array, Docker management, shares — recovers cleanly but takes longer
|
||||
restart_emhttp() {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart emhttp"
|
||||
return 0
|
||||
fi
|
||||
|
||||
info "$ICON_WEBGUI Restarting emhttp..."
|
||||
if /etc/rc.d/rc.emhttp restart >/dev/null 2>&1; then
|
||||
success "emhttp restarted"
|
||||
return 0
|
||||
else
|
||||
error "emhttp restart failed"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_WEBGUI WebGUI Watchdog ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
RECOVERY_ACTION=""
|
||||
RECOVERY_OK=false
|
||||
|
||||
# Initial check
|
||||
info "Checking WebGUI..."
|
||||
log "WebGUI check — $WEBGUI_URL"
|
||||
|
||||
# ── Healthy — completely silent ───────────────────────────────────────────────────────────────
|
||||
if check_webgui; then
|
||||
success "$ICON_WEBGUI WebGUI is responding — nothing to do"
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
|
||||
echo "$ICON_WEBGUI Status: $ICON_RUNNING HEALTHY"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
log "WebGUI responding — healthy ✅"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# WebGUI not responding — begin escalation
|
||||
warn "$ICON_WEBGUI WebGUI is not responding at $WEBGUI_URL"
|
||||
|
||||
# ── Step 1: Restart nginx ──
|
||||
# ── Not responding — begin escalation ────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_WEBGUI Step 1 — Nginx Restart ━━━"
|
||||
echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
warn "WebGUI not responding at $WEBGUI_URL — beginning escalation"
|
||||
|
||||
restart_nginx
|
||||
# ── Step 1 — nginx restart ────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 1 — nginx Restart ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
info "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..."
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart nginx"
|
||||
else
|
||||
warn "Restarting nginx..."
|
||||
if /etc/rc.d/rc.nginx restart >/dev/null 2>&1; then
|
||||
# Verify nginx actually running
|
||||
sleep 2
|
||||
if pgrep -x nginx >/dev/null 2>&1; then
|
||||
warn "nginx restarted ✅"
|
||||
else
|
||||
error "nginx not running after restart command"
|
||||
fi
|
||||
else
|
||||
error "nginx restart command failed"
|
||||
fi
|
||||
|
||||
log "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..."
|
||||
sleep "$WEBGUI_NGINX_WAIT"
|
||||
|
||||
if check_webgui; then
|
||||
success "$ICON_WEBGUI WebGUI recovered after nginx restart"
|
||||
notify "WebGUI recovered on $(hostname) after nginx restart" "WebGUI Watchdog" "warning"
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
|
||||
echo "$ICON_WEBGUI Status: $ICON_SUCCESS RECOVERED via nginx restart"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
RECOVERY_ACTION="nginx restart"
|
||||
RECOVERY_OK=true
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Step 2 — php-fpm restart ──────────────────────────────────────────────────────────────────
|
||||
if [[ "$RECOVERY_OK" == false ]]; then
|
||||
echo ""
|
||||
echo "━━━ Step 2 — php-fpm Restart ━━━"
|
||||
warn "WebGUI still not responding — restarting php-fpm"
|
||||
warn "WebGUI may be frozen due to worker exhaustion (check system_tuning_monitor.sh)"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart php-fpm"
|
||||
else
|
||||
if /etc/rc.d/rc.php-fpm restart >/dev/null 2>&1; then
|
||||
sleep 2
|
||||
if pgrep -f "php-fpm" >/dev/null 2>&1; then
|
||||
warn "php-fpm restarted ✅"
|
||||
else
|
||||
error "php-fpm not running after restart command"
|
||||
fi
|
||||
else
|
||||
error "php-fpm restart command failed"
|
||||
fi
|
||||
|
||||
warn "WebGUI still not responding after nginx restart — escalating to emhttp"
|
||||
log "Waiting ${WEBGUI_PHP_WAIT:-10}s for php-fpm to recover..."
|
||||
sleep "${WEBGUI_PHP_WAIT:-10}"
|
||||
|
||||
if check_webgui; then
|
||||
RECOVERY_ACTION="php-fpm restart"
|
||||
RECOVERY_OK=true
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Step 2: Restart emhttp ──
|
||||
echo ""
|
||||
echo "━━━ $ICON_WEBGUI Step 2 — emhttp Restart ━━━"
|
||||
warn "Restarting emhttp — this is the unRAID management daemon"
|
||||
warn "Array, Docker management and shares remain running but WebGUI will be briefly unavailable"
|
||||
|
||||
restart_emhttp
|
||||
# ── Step 3 — emhttp restart ───────────────────────────────────────────────────────────────────
|
||||
if [[ "$RECOVERY_OK" == false ]]; then
|
||||
echo ""
|
||||
echo "━━━ Step 3 — emhttp Restart ━━━"
|
||||
warn "WebGUI still not responding — restarting emhttp (unRAID management daemon)"
|
||||
warn "Array, Docker, and shares remain running — WebGUI management will briefly restart"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
info "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..."
|
||||
sleep "$WEBGUI_EMHTTP_WAIT"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart emhttp"
|
||||
else
|
||||
if /etc/rc.d/rc.emhttp restart >/dev/null 2>&1; then
|
||||
sleep 2
|
||||
if pgrep -x emhttp >/dev/null 2>&1; then
|
||||
warn "emhttp restarted ✅"
|
||||
else
|
||||
error "emhttp not running after restart command"
|
||||
fi
|
||||
else
|
||||
error "emhttp restart command failed"
|
||||
fi
|
||||
|
||||
if check_webgui; then
|
||||
success "$ICON_WEBGUI WebGUI recovered after emhttp restart"
|
||||
notify "WebGUI recovered on $(hostname) after emhttp restart — check system health" "WebGUI Watchdog" "warning"
|
||||
log "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..."
|
||||
sleep "$WEBGUI_EMHTTP_WAIT"
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
|
||||
echo "$ICON_WEBGUI Status: $ICON_SUCCESS RECOVERED via emhttp restart"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
if check_webgui; then
|
||||
RECOVERY_ACTION="emhttp restart"
|
||||
RECOVERY_OK=true
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Both restarts failed ──
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
|
||||
echo "$ICON_WEBGUI Status: $ICON_ERROR UNRECOVERED — manual intervention needed"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
notify "WebGUI unrecovered on $(hostname) after nginx and emhttp restart — manual intervention needed" "WebGUI Watchdog" "warning"
|
||||
exit 1
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no services restarted"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$RECOVERY_OK" == true ]]; then
|
||||
warn "$ICON_SUCCESS WebGUI recovered via: $RECOVERY_ACTION"
|
||||
notify "WebGUI recovered on $(hostname) ($MY_ID) via $RECOVERY_ACTION — monitor for recurrence" \
|
||||
"WebGUI Watchdog" "warning"
|
||||
else
|
||||
echo "$ICON_ERROR Status: UNRECOVERED — all three restart steps failed"
|
||||
echo "$ICON_ERROR Manual intervention needed:"
|
||||
echo " 1. Check: pgrep -x nginx emhttp"
|
||||
echo " 2. Check: journalctl -u nginx --since '10 minutes ago'"
|
||||
echo " 3. Try: server_reboot.sh if nothing else works"
|
||||
notify "WebGUI UNRECOVERED on $(hostname) ($MY_ID) — nginx + php-fpm + emhttp restart all failed — manual intervention needed" \
|
||||
"WebGUI Watchdog" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$RECOVERY_OK" == false && "$DRY_RUN" == false ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user