#!/bin/bash
# ==============================================================================================
# ============================ Watchdog Orchestrator ===========================================
# ==============================================================================================
# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
# Schedule: */15 * * * * (every 15 minutes)
#
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
#   Driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf — add, remove, or reorder there.
#   Default: resource_watchdog → docker_watchdog → system_watchdog → unraid_api_key_renew → stability_watchdog
#
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
#   Resource Watchdog first — frees RAM and CPU before healing attempts container restarts.
#                            Containers restarted into a resource-pressured system just fail again.
#   Docker Watchdog second — restarts with pressure already reduced, more likely to stabilise.
#   System Watchdog third  — system component health after containers are healed.
#   API key renew fourth   — self-heals unraid-api registry loss; check-first, silent when valid.
#   Stability Watchdog last — only reboots when all prior layers could not resolve the issue.
#
# ── ARRAY CHECK ───────────────────────────────────────────────────────────────────────────────
#   Exits immediately if /mnt/user is not mounted as shfs (array not started).
#   Watchdogs check Docker containers and storage — meaningless without the array.
#   Prevents false positives and unnecessary reboots when array is stopped or stopping.
#
# ── STARTUP GRACE ─────────────────────────────────────────────────────────────────────────────
#   No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds.
#   Prevents false positives from containers still starting at array launch.
#   Each sub-script enforces this independently — orchestrator exits early to avoid log noise.
#
# ── OVERLAP PROTECTION ────────────────────────────────────────────────────────────────────────
#   acquire_lock() — exits immediately if a prior cycle is still in progress.
#   Prevents pile-up when a cycle runs long (daemon restart attempt = 30s, etc.).
#
# ── REPLACES ──────────────────────────────────────────────────────────────────────────────────
#   Continuous loops previously in system_watchdog.sh and docker_watchdog.sh.
#   Those scripts are now single-pass — this orchestrator provides the cadence.
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   WATCHDOG_ORCHESTRATOR_SCRIPTS         — watchdogs to run, in order
#   WATCHDOG_STARTUP_GRACE                — seconds after boot before checks activate
#   WATCHDOG_ORCHESTRATOR_HEARTBEAT       — periodic heartbeat log toggle
#   WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   watchdog_orchestrator.sh           — normal run (called by cron every 15 minutes)
#   watchdog_orchestrator.sh --dry-run — pass --dry-run to all sub-scripts
#   watchdog_orchestrator.sh --status  — show script paths and current grace state
#   watchdog_orchestrator.sh --log     — verbose output from all sub-scripts
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

source "$ECOSYSTEM_ROOT/load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

# Skip immediately if another cycle is still running — no pile-up
acquire_lock

detect_hosts

log "$ICON_GEAR Config: grace=${WATCHDOG_STARTUP_GRACE}s heartbeat=${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}/${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr scripts=${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"
log "$ICON_WATCHDOG Order: $(for s in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do printf '%s ' "${s##*/}"; done)"

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"

# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
_watchdog_display_name() {
    local path="$1"
    local base="${path##*/}"
    base="${base%.sh}"
    base="${base//_/ }"
    echo "$base" | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2); print}'
}

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY WATCHDOG ORCHESTRATOR STATUS ━━━━━"
    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    echo ""

    UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
    if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
        warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)"
    else
        echo "Past startup grace — $(format_duration $UPTIME_S) uptime"
    fi

    echo ""
    echo "── Sub-scripts ──"
    for entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
        local_path="$ECOSYSTEM_ROOT/$entry"
        label="$(_watchdog_display_name "$entry")"
        if [[ -f "$local_path" ]]; then
            [[ -x "$local_path" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
            echo "  $icon $label — ${local_path##*/}"
        else
            echo "  $ICON_ERROR $label — NOT FOUND: $local_path"
        fi
    done

    echo ""
    echo "  Schedule:   */15 * * * * (every 15 minutes)"
    echo "  Heartbeat:  ${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true} / every ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Array Check ━━━
# ==============================================================================================
if ! platform_storage_healthy; then
    echo "Array not started — skipping watchdog cycle"
    exit 0
fi
log "$ICON_DISK Array: /mnt/user mounted (shfs) ✅"

# ==============================================================================================
# ━━━ Startup Grace ━━━
# ==============================================================================================
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
    echo "Startup grace — $(format_duration $UPTIME_SECONDS) / $(format_duration $WATCHDOG_STARTUP_GRACE) — skipping cycle"
    exit 0
fi
log "Startup grace: past — uptime $(format_duration $UPTIME_SECONDS)"

# ==============================================================================================
# ━━━ Run Watchdog Cycle ━━━
# ==============================================================================================
CYCLE_START=$(date +%s)
PASS=()
FAIL=()

run_watchdog() {
    local name="$1" script="$2"

    if [[ ! -f "$script" ]]; then
        error "$name — not found: $script"
        FAIL+=("$name:missing")
        return 1
    fi

    [[ ! -x "$script" ]] && chmod +x "$script"

    local extra_args=()
    [[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
    [[ "$VERBOSE" == true ]] && extra_args+=("--log")

    local _ws
    _ws=$(date +%s)
    log "$ICON_START $name"
    if bash "$script" "${extra_args[@]}"; then
        log "$ICON_DONE $name — done in $(format_duration $(( $(date +%s) - _ws )))"
        PASS+=("$name")
        return 0
    else
        error "$name — non-zero exit ($(format_duration $(( $(date +%s) - _ws ))))"
        FAIL+=("$name")
        return 1
    fi
}

for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
    run_watchdog "$(_watchdog_display_name "$_entry")" "$ECOSYSTEM_ROOT/$_entry"
done

CYCLE_END=$(date +%s)
DURATION=$(( CYCLE_END - CYCLE_START ))

# ==============================================================================================
# ━━━ Heartbeat ━━━
# ==============================================================================================
if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then
    HB_SECONDS=$(( ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1} * 3600 ))
    HB_COUNT_FILE="${STATE_DIR:-/tmp}/watchdog_orch_hb.count"
    HB_COUNT=$(cat "$HB_COUNT_FILE" 2>/dev/null || echo 0)
    HB_COUNT=$(( HB_COUNT + 1 ))
    echo "$HB_COUNT" > "$HB_COUNT_FILE"
    # Each cron run = ~60s — use count × 60 as uptime approximation
    HB_ELAPSED=$(( HB_COUNT * 60 ))
    if [[ "$HB_SECONDS" -gt 0 ]] && (( HB_ELAPSED % HB_SECONDS < 60 )) && [[ "$HB_COUNT" -gt 1 ]]; then
        HB_HR=$(( HB_ELAPSED / 3600 ))
        warn "♥ watchdog_orchestrator alive — $MY_ID — ~${HB_HR}hr ($(date '+%H:%M:%S'))"
    fi
fi

# ==============================================================================================
# ━━━ Summary — only shown on failures or --log ━━━
# ==============================================================================================
if [[ "${#FAIL[@]}" -gt 0 || "$VERBOSE" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
    for p in "${PASS[@]}"; do log "  $ICON_DONE  $p"; done
    for f in "${FAIL[@]}"; do error "  $ICON_ERROR $f"; done
    echo "$ICON_TIME  Duration: $(format_duration $DURATION)"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

    if [[ "${#FAIL[@]}" -gt 0 ]]; then
        notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
            "Watchdog Orchestrator" "warning"
    fi
fi