#!/bin/bash # ----------------------------------------------------------------------------------------------- # --------------------------------- WebGUI Watchdog -------------------------------------------- # ----------------------------------------------------------------------------------------------- # Monitors unRAID's WebGUI and restarts it if unresponsive. # Uses an escalating restart strategy — tries nginx first, then emhttp if needed. # emhttp is the unRAID management daemon — restarting it is more disruptive than nginx # but recovers cleanly. Notification sent on any restart so you know what happened. # # Escalation path: # Check WebGUI → unresponsive → restart nginx → recheck # Still unresponsive → restart emhttp → recheck # Still unresponsive → notify warning, manual intervention needed # # Run every 5-10 minutes via cron/User Scripts plugin. # All configuration in Master.conf under WebGUI Watchdog section. # Supports --dry-run to show what would be restarted without acting. # ----------------------------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../common.sh" parse_args "$@" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Setup ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_GEAR Setup ━━━" if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi success "Running as root" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Status ━━━ # ----------------------------------------------------------------------------------------------- if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "$ICON_WEBGUI URL: $WEBGUI_URL" echo "$ICON_WEBGUI Curl timeout: ${WEBGUI_TIMEOUT}s" echo "$ICON_WEBGUI Nginx wait: ${WEBGUI_NGINX_WAIT}s" echo "$ICON_WEBGUI emhttp wait: ${WEBGUI_EMHTTP_WAIT}s" echo "$ICON_GEAR Dry Run: $DRY_RUN" # Show current state if curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1; then echo "$ICON_WEBGUI WebGUI: $ICON_RUNNING responding" else echo "$ICON_WEBGUI WebGUI: $ICON_NOT_RUNNING not responding" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted" # ----------------------------------------------------------------------------------------------- # FUNCTIONS # ----------------------------------------------------------------------------------------------- # Check if WebGUI is responding check_webgui() { curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1 } # Restart nginx — lightweight fix, try first restart_nginx() { if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would restart nginx" return 0 fi info "$ICON_WEBGUI Restarting nginx..." if /etc/rc.d/rc.nginx restart >/dev/null 2>&1; then success "nginx restarted" return 0 else error "nginx restart failed" return 1 fi } # Restart emhttp — heavier fix, escalate if nginx didn't help # emhttp drives the array, Docker management, shares — recovers cleanly but takes longer restart_emhttp() { if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would restart emhttp" return 0 fi info "$ICON_WEBGUI Restarting emhttp..." if /etc/rc.d/rc.emhttp restart >/dev/null 2>&1; then success "emhttp restarted" return 0 else error "emhttp restart failed" return 1 fi } # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_WEBGUI WebGUI Watchdog ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_WEBGUI URL: $WEBGUI_URL" echo "" START=$(date +%s) # Initial check info "Checking WebGUI..." if check_webgui; then success "$ICON_WEBGUI WebGUI is responding — nothing to do" echo "" echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━" echo "$ICON_WEBGUI Status: $ICON_RUNNING HEALTHY" echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # WebGUI not responding — begin escalation warn "$ICON_WEBGUI WebGUI is not responding at $WEBGUI_URL" # ── Step 1: Restart nginx ── echo "" echo "━━━ $ICON_WEBGUI Step 1 — Nginx Restart ━━━" restart_nginx if [[ "$DRY_RUN" == false ]]; then info "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..." sleep "$WEBGUI_NGINX_WAIT" if check_webgui; then success "$ICON_WEBGUI WebGUI recovered after nginx restart" notify "WebGUI recovered on $(hostname) after nginx restart" "WebGUI Watchdog" "warning" echo "" echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━" echo "$ICON_WEBGUI Status: $ICON_SUCCESS RECOVERED via nginx restart" echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi warn "WebGUI still not responding after nginx restart — escalating to emhttp" fi # ── Step 2: Restart emhttp ── echo "" echo "━━━ $ICON_WEBGUI Step 2 — emhttp Restart ━━━" warn "Restarting emhttp — this is the unRAID management daemon" warn "Array, Docker management and shares remain running but WebGUI will be briefly unavailable" restart_emhttp if [[ "$DRY_RUN" == false ]]; then info "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..." sleep "$WEBGUI_EMHTTP_WAIT" if check_webgui; then success "$ICON_WEBGUI WebGUI recovered after emhttp restart" notify "WebGUI recovered on $(hostname) after emhttp restart — check system health" "WebGUI Watchdog" "warning" echo "" echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━" echo "$ICON_WEBGUI Status: $ICON_SUCCESS RECOVERED via emhttp restart" echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi fi # ── Both restarts failed ── END=$(date +%s) echo "" echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━" echo "$ICON_WEBGUI Status: $ICON_ERROR UNRECOVERED — manual intervention needed" echo "$ICON_TIME Duration: $(format_duration $((END - START)))" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" if [[ "$DRY_RUN" == false ]]; then notify "WebGUI unrecovered on $(hostname) after nginx and emhttp restart — manual intervention needed" "WebGUI Watchdog" "warning" exit 1 fi exit 0