Varaverk/unRAID_Essentials/webgui_restart.sh

#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- WebGUI Watchdog --------------------------------------------
# -----------------------------------------------------------------------------------------------
# Monitors unRAID's WebGUI and restarts it if unresponsive.
# Uses an escalating restart strategy — tries nginx first, then emhttp if needed.
# emhttp is the unRAID management daemon — restarting it is more disruptive than nginx
# but recovers cleanly. Notification sent on any restart so you know what happened.
#
# Escalation path:
#   Check WebGUI → unresponsive → restart nginx → recheck
#   Still unresponsive → restart emhttp → recheck
#   Still unresponsive → notify warning, manual intervention needed
#
# Run every 5-10 minutes via cron/User Scripts plugin.
# All configuration in Master.conf under WebGUI Watchdog section.
# Supports --dry-run to show what would be restarted without acting.
# -----------------------------------------------------------------------------------------------

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"

parse_args "$@"

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

success "Running as root"

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_WEBGUI URL:           $WEBGUI_URL"
    echo "$ICON_WEBGUI Curl timeout:  ${WEBGUI_TIMEOUT}s"
    echo "$ICON_WEBGUI Nginx wait:    ${WEBGUI_NGINX_WAIT}s"
    echo "$ICON_WEBGUI emhttp wait:   ${WEBGUI_EMHTTP_WAIT}s"
    echo "$ICON_GEAR   Dry Run:       $DRY_RUN"

    # Show current state
    if curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1; then
        echo "$ICON_WEBGUI WebGUI:       $ICON_RUNNING responding"
    else
        echo "$ICON_WEBGUI WebGUI:       $ICON_NOT_RUNNING not responding"
    fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted"

# -----------------------------------------------------------------------------------------------
# FUNCTIONS
# -----------------------------------------------------------------------------------------------

# Check if WebGUI is responding
check_webgui() {
    curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1
}

# Restart nginx — lightweight fix, try first
restart_nginx() {
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would restart nginx"
        return 0
    fi

    info "$ICON_WEBGUI Restarting nginx..."
    if /etc/rc.d/rc.nginx restart >/dev/null 2>&1; then
        success "nginx restarted"
        return 0
    else
        error "nginx restart failed"
        return 1
    fi
}

# Restart emhttp — heavier fix, escalate if nginx didn't help
# emhttp drives the array, Docker management, shares — recovers cleanly but takes longer
restart_emhttp() {
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would restart emhttp"
        return 0
    fi

    info "$ICON_WEBGUI Restarting emhttp..."
    if /etc/rc.d/rc.emhttp restart >/dev/null 2>&1; then
        success "emhttp restarted"
        return 0
    else
        error "emhttp restart failed"
        return 1
    fi
}

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_WEBGUI WebGUI Watchdog ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
echo ""

START=$(date +%s)

# Initial check
info "Checking WebGUI..."

if check_webgui; then
    success "$ICON_WEBGUI WebGUI is responding — nothing to do"
    echo ""
    echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
    echo "$ICON_WEBGUI Status:   $ICON_RUNNING HEALTHY"
    echo "$ICON_TIME   Duration: $(format_duration $(($(date +%s) - START)))"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# WebGUI not responding — begin escalation
warn "$ICON_WEBGUI WebGUI is not responding at $WEBGUI_URL"

# ── Step 1: Restart nginx ──
echo ""
echo "━━━ $ICON_WEBGUI Step 1 — Nginx Restart ━━━"

restart_nginx

if [[ "$DRY_RUN" == false ]]; then
    info "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..."
    sleep "$WEBGUI_NGINX_WAIT"

    if check_webgui; then
        success "$ICON_WEBGUI WebGUI recovered after nginx restart"
        notify "WebGUI recovered on $(hostname) after nginx restart" "WebGUI Watchdog" "warning"

        echo ""
        echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
        echo "$ICON_WEBGUI Status:   $ICON_SUCCESS RECOVERED via nginx restart"
        echo "$ICON_TIME   Duration: $(format_duration $(($(date +%s) - START)))"
        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
        exit 0
    fi

    warn "WebGUI still not responding after nginx restart — escalating to emhttp"
fi

# ── Step 2: Restart emhttp ──
echo ""
echo "━━━ $ICON_WEBGUI Step 2 — emhttp Restart ━━━"
warn "Restarting emhttp — this is the unRAID management daemon"
warn "Array, Docker management and shares remain running but WebGUI will be briefly unavailable"

restart_emhttp

if [[ "$DRY_RUN" == false ]]; then
    info "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..."
    sleep "$WEBGUI_EMHTTP_WAIT"

    if check_webgui; then
        success "$ICON_WEBGUI WebGUI recovered after emhttp restart"
        notify "WebGUI recovered on $(hostname) after emhttp restart — check system health" "WebGUI Watchdog" "warning"

        echo ""
        echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
        echo "$ICON_WEBGUI Status:   $ICON_SUCCESS RECOVERED via emhttp restart"
        echo "$ICON_TIME   Duration: $(format_duration $(($(date +%s) - START)))"
        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
        exit 0
    fi
fi

# ── Both restarts failed ──
END=$(date +%s)

echo ""
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
echo "$ICON_WEBGUI Status:   $ICON_ERROR UNRECOVERED — manual intervention needed"
echo "$ICON_TIME   Duration: $(format_duration $((END - START)))"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

if [[ "$DRY_RUN" == false ]]; then
    notify "WebGUI unrecovered on $(hostname) after nginx and emhttp restart — manual intervention needed" "WebGUI Watchdog" "warning"
    exit 1
fi

exit 0