#!/bin/bash
# ==============================================================================================
# ================================= System Watchdog ============================================
# ==============================================================================================
# Thin orchestrator — runs SYSTEM_WATCHDOG_SCRIPTS from master.conf sequentially.
# Called by watchdog_orchestrator.sh each cycle. Covers system component health:
# storage pool growth, runaway logs, and WebGUI availability.
#
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
#   Driven by SYSTEM_WATCHDOG_SCRIPTS in master.conf — add, remove, or reorder there.
#   Default: storage_watchdog → webgui_watchdog
#
# ── SEQUENTIAL EXECUTION ─────────────────────────────────────────────────────────────────────
# All scripts run in the foreground. Each must complete before the next starts.
# A failed script is logged but does not prevent remaining scripts from running.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
#   Root check           — child scripts require root
#   acquire_lock         — prevents concurrent system watchdog runs
#   detect_hosts()       — MY_ID in notifications and logs
#   Non-fatal steps      — a failed step is logged; remaining steps still run
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   SYSTEM_WATCHDOG_SCRIPTS — ordered list of system component watchdog scripts to run
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   system_watchdog.sh              — run all system component watchdogs
#   system_watchdog.sh --dry-run    — preview without running anything
#   system_watchdog.sh --status     — show configured scripts and exit
#   system_watchdog.sh --log        — verbose output
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

source "$ECOSYSTEM_ROOT/load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

validate_unraid_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

acquire_lock

detect_hosts

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no watchdog scripts will be executed"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY SYSTEM WATCHDOG STATUS ━━━━━"
    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR  Scripts:  ${#SYSTEM_WATCHDOG_SCRIPTS[@]} configured"
    echo ""

    for entry in "${SYSTEM_WATCHDOG_SCRIPTS[@]}"; do
        [[ -z "$entry" ]] && continue
        read -r -a parts <<< "$entry"
        script_path="$ECOSYSTEM_ROOT/${parts[0]}"
        script_name=$(basename "${parts[0]}")
        if [[ ! -f "$script_path" ]]; then
            echo "  $ICON_ERROR $script_name — FILE NOT FOUND"
        else
            echo "  $ICON_GEAR $script_name"
        fi
    done
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Run Sequence ━━━
# ==============================================================================================
START=$(date +%s)
PASSED=()
FAILED=()
STEP=0

for entry in "${SYSTEM_WATCHDOG_SCRIPTS[@]}"; do
    [[ -z "$entry" ]] && continue
    (( STEP++ ))

    read -r -a parts <<< "$entry"
    script_path="$ECOSYSTEM_ROOT/${parts[0]}"
    script_name=$(basename "${parts[0]}")

    if [[ ! -f "$script_path" ]]; then
        error "$script_name — not found at $script_path"
        FAILED+=("$script_name")
        continue
    fi

    if [[ ! -x "$script_path" ]]; then
        chmod +x "$script_path" || {
            error "$script_name — chmod +x failed"
            FAILED+=("$script_name")
            continue
        }
    fi

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would run: $script_name"
        PASSED+=("$script_name")
        continue
    fi

    if bash "$script_path"; then
        log "$script_name — done ✅"
        PASSED+=("$script_name")
    else
        warn "$script_name — exit non-zero (issues found or fixed) — continuing"
        FAILED+=("$script_name")
    fi
done

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
log "System watchdog — $STEP script(s) — $(format_duration $(( END - START )))"

[[ ${#FAILED[@]} -gt 0 ]] && exit 1
exit 0