#!/bin/bash
# ==============================================================================================
# ================================= WebGUI Watchdog ============================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Monitors the unRAID WebGUI and restarts services if unresponsive. Uses a
# three-step escalating strategy — lightest fix first, heaviest last. Called
# by system_watchdog.sh each cycle. Silent when healthy.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Escalation Path
#   WebGUI responding → log() + exit 0 (completely silent ✅)
#
#   Not responding:
#     Step 1 — nginx restart
#               Lightest fix — handles most transient WebGUI failures:
#               nginx crash, worker stuck, connection timeout.
#               Wait WEBGUI_NGINX_WAIT seconds → recheck.
#
#     Step 2 — php-fpm restart
#               WebGUI runs through PHP-FPM. Worker exhaustion causes silent
#               failure — requests queue and the WebGUI appears frozen.
#               Wait WEBGUI_PHP_WAIT seconds → recheck.
#
#     Step 3 — emhttp restart
#               Heaviest fix. emhttp is the unRAID management daemon.
#               Array, Docker, and shares stay running — only WebGUI
#               management restarts. Takes longer — WEBGUI_EMHTTP_WAIT.
#               Wait WEBGUI_EMHTTP_WAIT seconds → recheck.
#
#     All three failed → notify, manual intervention needed → exit 1.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root Required
#   Service restart commands require root.
#
# Single Instance Lock
#   acquire_lock prevents concurrent runs double-restarting services.
#
# Process Verify After Each Restart
#   pgrep check after each rc.* command — errors if process not running.
#
# Silent When Healthy
#   Completely silent on healthy cycles. Only produces output when recovering.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   WEBGUI_URL
#       URL to check for WebGUI response. (default: http://localhost)
#
#   WEBGUI_TIMEOUT
#       curl timeout in seconds. (default: 5)
#
#   WEBGUI_NGINX_WAIT
#       Seconds after nginx restart before rechecking. (default: 15)
#
#   WEBGUI_PHP_WAIT
#       Seconds after php-fpm restart before rechecking. (default: 10)
#
#   WEBGUI_EMHTTP_WAIT
#       Seconds after emhttp restart before rechecking. (default: 30)
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# webgui_watchdog.sh
#     Check WebGUI. Escalate through nginx → php-fpm → emhttp if unresponsive.
#
# webgui_watchdog.sh --dry-run
#     Show which services would be restarted. No restarts, no waits.
#
# webgui_watchdog.sh --status
#     Show current WebGUI response state and nginx/php-fpm/emhttp process states.
#
# webgui_watchdog.sh --log
#     Verbose output — show each check, each restart attempt, each wait.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

platform_require_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

acquire_lock

detect_hosts

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG STATUS ━━━━━"
    echo "$ICON_HOST    Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_WEBGUI  URL:         $WEBGUI_URL"
    echo "$ICON_WEBGUI  Timeouts:    curl=${WEBGUI_TIMEOUT}s nginx=${WEBGUI_NGINX_WAIT}s php=${WEBGUI_PHP_WAIT:-10}s emhttp=${WEBGUI_EMHTTP_WAIT}s"
    echo ""

    if curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1; then
        echo "  $ICON_SUCCESS WebGUI:    responding ✅"
    else
        echo "  $ICON_ERROR  WebGUI:    NOT responding"
    fi

    platform_is_service_running nginx && \
        echo "  $ICON_SUCCESS nginx:     running ✅" || \
        echo "  $ICON_ERROR  nginx:     NOT running"

    platform_is_service_running php-fpm && \
        FPM_COUNT=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?") && \
        echo "  $ICON_SUCCESS php-fpm:   running ($FPM_COUNT workers) ✅" || \
        echo "  $ICON_ERROR  php-fpm:   NOT running"

    platform_is_service_running emhttp && \
        echo "  $ICON_SUCCESS emhttp:    running ✅" || \
        echo "  $ICON_ERROR  emhttp:    NOT running"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ── CHECK AND ESCALATE ────────────────────────────────────────────────────────────────────────
# ==============================================================================================
check_webgui() {
    curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1
}

START=$(date +%s)
RECOVERY_ACTION=""
RECOVERY_OK=false

log "WebGUI check — $WEBGUI_URL"

# ── Healthy — completely silent ───────────────────────────────────────────────────────────────
if check_webgui; then
    _nginx_count=$(pgrep -cx nginx 2>/dev/null || echo 0)
    _fpm_count=$(pgrep -fc "php-fpm" 2>/dev/null || echo 0)
    log "$ICON_WEBGUI WebGUI responding ✅ — nginx workers:${_nginx_count} php-fpm workers:${_fpm_count}"
    echo "WebGUI responding — healthy ✅"
    exit 0
fi

# ── Not responding — begin escalation ────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
echo ""
warn "WebGUI not responding at $WEBGUI_URL — beginning escalation"

# ── Step 1 — nginx restart ────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ Step 1 — nginx Restart ━━━"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would restart nginx"
else
    warn "Restarting nginx..."
    if platform_restart_service nginx; then
        # Verify nginx actually running
        sleep 2
        if platform_is_service_running nginx; then
            warn "nginx restarted ✅"
        else
            error "nginx not running after restart command"
        fi
    else
        error "nginx restart command failed"
    fi

    log "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..."
    sleep "$WEBGUI_NGINX_WAIT"

    if check_webgui; then
        RECOVERY_ACTION="nginx restart"
        RECOVERY_OK=true
    fi
fi

# ── Step 2 — php-fpm restart ──────────────────────────────────────────────────────────────────
if [[ "$RECOVERY_OK" == false ]]; then
    echo ""
    echo "━━━ Step 2 — php-fpm Restart ━━━"
    warn "WebGUI still not responding — restarting php-fpm"
    warn "WebGUI may be frozen due to worker exhaustion (check system_tuning_monitor.sh)"

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would restart php-fpm"
    else
        if platform_restart_service php-fpm; then
            sleep 2
            if platform_is_service_running php-fpm; then
                warn "php-fpm restarted ✅"
            else
                error "php-fpm not running after restart command"
            fi
        else
            error "php-fpm restart command failed"
        fi

        log "Waiting ${WEBGUI_PHP_WAIT:-10}s for php-fpm to recover..."
        sleep "${WEBGUI_PHP_WAIT:-10}"

        if check_webgui; then
            RECOVERY_ACTION="php-fpm restart"
            RECOVERY_OK=true
        fi
    fi
fi

# ── Step 3 — emhttp restart ───────────────────────────────────────────────────────────────────
if [[ "$RECOVERY_OK" == false ]]; then
    echo ""
    echo "━━━ Step 3 — emhttp Restart ━━━"
    warn "WebGUI still not responding — restarting emhttp (unRAID management daemon)"
    warn "Array, Docker, and shares remain running — WebGUI management will briefly restart"

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would restart emhttp"
    else
        if /usr/local/sbin/emhttp stop >/dev/null 2>&1 && /usr/local/sbin/emhttp start >/dev/null 2>&1; then
            sleep 2
            if platform_is_service_running emhttp; then
                warn "emhttp restarted ✅"
            else
                error "emhttp not running after restart command"
            fi
        else
            error "emhttp restart command failed"
        fi

        log "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..."
        sleep "$WEBGUI_EMHTTP_WAIT"

        if check_webgui; then
            RECOVERY_ACTION="emhttp restart"
            RECOVERY_OK=true
        fi
    fi
fi

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
echo "$ICON_HOST    Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_WEBGUI  URL:      $WEBGUI_URL"
echo "$ICON_TIME    Duration: $(format_duration $(( END - START )))"
echo ""

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no services restarted"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

if [[ "$RECOVERY_OK" == true ]]; then
    warn "$ICON_SUCCESS WebGUI recovered via: $RECOVERY_ACTION"
    notify "WebGUI recovered on $(hostname) ($MY_ID) via $RECOVERY_ACTION — monitor for recurrence" \
        "WebGUI Watchdog" "warning"
else
    echo "$ICON_ERROR Status: UNRECOVERED — all three restart steps failed"
    echo "$ICON_ERROR Manual intervention needed:"
    echo "    1. Check: pgrep nginx; pgrep emhttpd"
    echo "    2. Check: journalctl -u nginx --since '10 minutes ago'"
    echo "    3. Try: server_reboot.sh if nothing else works"
    notify "WebGUI UNRECOVERED on $(hostname) ($MY_ID) — nginx + php-fpm + emhttp restart all failed — manual intervention needed" \
        "WebGUI Watchdog" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ "$RECOVERY_OK" == false && "$DRY_RUN" == false ]] && exit 1
exit 0