Auth stack certs tab, arrs db fallbacks, cert monitor cache, conf parser fix
- Auth stack: fold cert monitor into Auth Stack page as fourth tab (Certs); remove standalone cert page and top-level tab - cert_monitor.sh: write JSON status cache to State_Files/cert_status.json after each run; expose per-domain days/expiry via _CERT_DAYS/_CERT_EXPIRY globals - api/cert.php: new — serves cached cert status; falls back to configured domains as UNKN when no cache exists; POST action=run triggers live check - arrs db fallbacks: vv_arr_cleanup_stats/discovery_stats/recovery_stats now read from data/*.db files when log JSON files don't yet exist - config.php vv_conf_vars(): unescape bash \$ → $ so passwords with dollar signs read correctly from conf files - host1.conf: fill in HOST1_NPM_USER/PASS and HOST1_LLDAP_USER/PASS - Partnership adapter pattern: Unraid-specific container logic extracted to Plugin/unraid/Partnership/; platform-agnostic structure stays in Partnership/ - First-run wizard: uniform multi-step flow for all hosts; HOST2 pull moved to checklist; auto SSH keygen and API key creation on save - api/checklist.php: live setup checklist with pull_master action - Fullscreen toggle: hide Unraid header/menu; state persists via localStorage
This commit is contained in:
Gmer4Lfe
@@ -1,305 +0,0 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= WebGUI Watchdog ============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Monitors the unRAID WebGUI and restarts services if unresponsive. Uses a
|
||||
# three-step escalating strategy — lightest fix first, heaviest last. Called
|
||||
# by system_watchdog.sh each cycle. Silent when healthy.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Escalation Path
|
||||
# WebGUI responding → log() + exit 0 (completely silent ✅)
|
||||
#
|
||||
# Not responding:
|
||||
# Step 1 — nginx restart
|
||||
# Lightest fix — handles most transient WebGUI failures:
|
||||
# nginx crash, worker stuck, connection timeout.
|
||||
# Wait WEBGUI_NGINX_WAIT seconds → recheck.
|
||||
#
|
||||
# Step 2 — php-fpm restart
|
||||
# WebGUI runs through PHP-FPM. Worker exhaustion causes silent
|
||||
# failure — requests queue and the WebGUI appears frozen.
|
||||
# Wait WEBGUI_PHP_WAIT seconds → recheck.
|
||||
#
|
||||
# Step 3 — emhttp restart
|
||||
# Heaviest fix. emhttp is the unRAID management daemon.
|
||||
# Array, Docker, and shares stay running — only WebGUI
|
||||
# management restarts. Takes longer — WEBGUI_EMHTTP_WAIT.
|
||||
# Wait WEBGUI_EMHTTP_WAIT seconds → recheck.
|
||||
#
|
||||
# All three failed → notify, manual intervention needed → exit 1.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Required
|
||||
# Service restart commands require root.
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents concurrent runs double-restarting services.
|
||||
#
|
||||
# Process Verify After Each Restart
|
||||
# pgrep check after each rc.* command — errors if process not running.
|
||||
#
|
||||
# Silent When Healthy
|
||||
# Completely silent on healthy cycles. Only produces output when recovering.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# WEBGUI_URL
|
||||
# URL to check for WebGUI response. (default: http://localhost)
|
||||
#
|
||||
# WEBGUI_TIMEOUT
|
||||
# curl timeout in seconds. (default: 5)
|
||||
#
|
||||
# WEBGUI_NGINX_WAIT
|
||||
# Seconds after nginx restart before rechecking. (default: 15)
|
||||
#
|
||||
# WEBGUI_PHP_WAIT
|
||||
# Seconds after php-fpm restart before rechecking. (default: 10)
|
||||
#
|
||||
# WEBGUI_EMHTTP_WAIT
|
||||
# Seconds after emhttp restart before rechecking. (default: 30)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# webgui_watchdog.sh
|
||||
# Check WebGUI. Escalate through nginx → php-fpm → emhttp if unresponsive.
|
||||
#
|
||||
# webgui_watchdog.sh --dry-run
|
||||
# Show which services would be restarted. No restarts, no waits.
|
||||
#
|
||||
# webgui_watchdog.sh --status
|
||||
# Show current WebGUI response state and nginx/php-fpm/emhttp process states.
|
||||
#
|
||||
# webgui_watchdog.sh --log
|
||||
# Verbose output — show each check, each restart attempt, each wait.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
platform_require_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
|
||||
echo "$ICON_WEBGUI Timeouts: curl=${WEBGUI_TIMEOUT}s nginx=${WEBGUI_NGINX_WAIT}s php=${WEBGUI_PHP_WAIT:-10}s emhttp=${WEBGUI_EMHTTP_WAIT}s"
|
||||
echo ""
|
||||
|
||||
if curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1; then
|
||||
echo " $ICON_SUCCESS WebGUI: responding ✅"
|
||||
else
|
||||
echo " $ICON_ERROR WebGUI: NOT responding"
|
||||
fi
|
||||
|
||||
platform_is_service_running nginx && \
|
||||
echo " $ICON_SUCCESS nginx: running ✅" || \
|
||||
echo " $ICON_ERROR nginx: NOT running"
|
||||
|
||||
platform_is_service_running php-fpm && \
|
||||
FPM_COUNT=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?") && \
|
||||
echo " $ICON_SUCCESS php-fpm: running ($FPM_COUNT workers) ✅" || \
|
||||
echo " $ICON_ERROR php-fpm: NOT running"
|
||||
|
||||
platform_is_service_running emhttp && \
|
||||
echo " $ICON_SUCCESS emhttp: running ✅" || \
|
||||
echo " $ICON_ERROR emhttp: NOT running"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── CHECK AND ESCALATE ────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
check_webgui() {
|
||||
curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
START=$(date +%s)
|
||||
RECOVERY_ACTION=""
|
||||
RECOVERY_OK=false
|
||||
|
||||
log "WebGUI check — $WEBGUI_URL"
|
||||
|
||||
# ── Healthy — completely silent ───────────────────────────────────────────────────────────────
|
||||
if check_webgui; then
|
||||
_nginx_count=$(pgrep -cx nginx 2>/dev/null || echo 0)
|
||||
_fpm_count=$(pgrep -fc "php-fpm" 2>/dev/null || echo 0)
|
||||
log "$ICON_WEBGUI WebGUI responding ✅ — nginx workers:${_nginx_count} php-fpm workers:${_fpm_count}"
|
||||
echo "WebGUI responding — healthy ✅"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Not responding — begin escalation ────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
warn "WebGUI not responding at $WEBGUI_URL — beginning escalation"
|
||||
|
||||
# ── Step 1 — nginx restart ────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 1 — nginx Restart ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart nginx"
|
||||
else
|
||||
warn "Restarting nginx..."
|
||||
if platform_restart_service nginx; then
|
||||
# Verify nginx actually running
|
||||
sleep 2
|
||||
if platform_is_service_running nginx; then
|
||||
warn "nginx restarted ✅"
|
||||
else
|
||||
error "nginx not running after restart command"
|
||||
fi
|
||||
else
|
||||
error "nginx restart command failed"
|
||||
fi
|
||||
|
||||
log "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..."
|
||||
sleep "$WEBGUI_NGINX_WAIT"
|
||||
|
||||
if check_webgui; then
|
||||
RECOVERY_ACTION="nginx restart"
|
||||
RECOVERY_OK=true
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Step 2 — php-fpm restart ──────────────────────────────────────────────────────────────────
|
||||
if [[ "$RECOVERY_OK" == false ]]; then
|
||||
echo ""
|
||||
echo "━━━ Step 2 — php-fpm Restart ━━━"
|
||||
warn "WebGUI still not responding — restarting php-fpm"
|
||||
warn "WebGUI may be frozen due to worker exhaustion (check system_tuning_monitor.sh)"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart php-fpm"
|
||||
else
|
||||
if platform_restart_service php-fpm; then
|
||||
sleep 2
|
||||
if platform_is_service_running php-fpm; then
|
||||
warn "php-fpm restarted ✅"
|
||||
else
|
||||
error "php-fpm not running after restart command"
|
||||
fi
|
||||
else
|
||||
error "php-fpm restart command failed"
|
||||
fi
|
||||
|
||||
log "Waiting ${WEBGUI_PHP_WAIT:-10}s for php-fpm to recover..."
|
||||
sleep "${WEBGUI_PHP_WAIT:-10}"
|
||||
|
||||
if check_webgui; then
|
||||
RECOVERY_ACTION="php-fpm restart"
|
||||
RECOVERY_OK=true
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Step 3 — emhttp restart ───────────────────────────────────────────────────────────────────
|
||||
if [[ "$RECOVERY_OK" == false ]]; then
|
||||
echo ""
|
||||
echo "━━━ Step 3 — emhttp Restart ━━━"
|
||||
warn "WebGUI still not responding — restarting emhttp (unRAID management daemon)"
|
||||
warn "Array, Docker, and shares remain running — WebGUI management will briefly restart"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart emhttp"
|
||||
else
|
||||
if /usr/local/sbin/emhttp stop >/dev/null 2>&1 && /usr/local/sbin/emhttp start >/dev/null 2>&1; then
|
||||
sleep 2
|
||||
if platform_is_service_running emhttp; then
|
||||
warn "emhttp restarted ✅"
|
||||
else
|
||||
error "emhttp not running after restart command"
|
||||
fi
|
||||
else
|
||||
error "emhttp restart command failed"
|
||||
fi
|
||||
|
||||
log "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..."
|
||||
sleep "$WEBGUI_EMHTTP_WAIT"
|
||||
|
||||
if check_webgui; then
|
||||
RECOVERY_ACTION="emhttp restart"
|
||||
RECOVERY_OK=true
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no services restarted"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$RECOVERY_OK" == true ]]; then
|
||||
warn "$ICON_SUCCESS WebGUI recovered via: $RECOVERY_ACTION"
|
||||
notify "WebGUI recovered on $(hostname) ($MY_ID) via $RECOVERY_ACTION — monitor for recurrence" \
|
||||
"WebGUI Watchdog" "warning"
|
||||
else
|
||||
echo "$ICON_ERROR Status: UNRECOVERED — all three restart steps failed"
|
||||
echo "$ICON_ERROR Manual intervention needed:"
|
||||
echo " 1. Check: pgrep nginx; pgrep emhttpd"
|
||||
echo " 2. Check: journalctl -u nginx --since '10 minutes ago'"
|
||||
echo " 3. Try: server_reboot.sh if nothing else works"
|
||||
notify "WebGUI UNRECOVERED on $(hostname) ($MY_ID) — nginx + php-fpm + emhttp restart all failed — manual intervention needed" \
|
||||
"WebGUI Watchdog" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$RECOVERY_OK" == false && "$DRY_RUN" == false ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user