#!/bin/bash
# ==============================================================================================
# ============================ Watchdog Orchestrator ===========================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. Replaces the
# continuous loops previously embedded in individual watchdog scripts — those
# are now single-pass; this orchestrator provides the cadence.
# Schedule: */15 * * * * (every 15 minutes)
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Order driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf.
# Default: resource_watchdog → docker_watchdog → system_watchdog →
#          unraid_api_key_renew → stability_watchdog
#
# ARRAY CHECK
#   Exits immediately if /mnt/user is not mounted as shfs. Watchdogs check
#   Docker containers and storage — meaningless without the array. Prevents
#   false positives and unnecessary reboots when array is stopped or stopping.
#
# STARTUP GRACE
#   No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds. Prevents
#   false positives from containers still starting at array launch. Each
#   sub-script enforces this independently — orchestrator exits early to avoid
#   log noise.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Pressure Before Healing
#   Resource Watchdog runs first — it frees RAM and CPU before any container
#   restart is attempted. Containers restarted into a resource-pressured system
#   just fail again. Docker Watchdog restarts with pressure already reduced.
#   System Watchdog checks component health after containers are healed.
#   Stability Watchdog reboots only when all prior layers could not resolve the
#   issue. API key renew is check-first and silent when valid.
#
# Never Queue
#   acquire_lock exits immediately if a prior cycle is still running. Prevents
#   pile-up when a cycle runs long (daemon restart attempt = 30s, etc.).
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
#   Root check       — watchdog operations require root
#   acquire_lock     — strict; no pile-up if prior cycle still active
#   detect_hosts()   — MY_ID in logs and notifications
#   Array check      — exits early if /mnt/user is not shfs-mounted
#   Startup grace    — WATCHDOG_STARTUP_GRACE respected before any checks
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   WATCHDOG_ORCHESTRATOR_SCRIPTS         — watchdogs to run, in order
#   WATCHDOG_STARTUP_GRACE                — seconds after boot before checks activate
#   WATCHDOG_ORCHESTRATOR_HEARTBEAT       — periodic heartbeat log toggle
#   WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# watchdog_orchestrator.sh
#     Normal run (called by cron every 15 minutes).
#
# watchdog_orchestrator.sh --dry-run
#     Pass --dry-run to all sub-scripts.
#
# watchdog_orchestrator.sh --status
#     Show script paths and current grace state.
#
# watchdog_orchestrator.sh --log
#     Verbose output from all sub-scripts.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

source "$ECOSYSTEM_ROOT/load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

# Skip immediately if another cycle is still running — no pile-up
acquire_lock

detect_hosts

log "$ICON_GEAR Config: grace=${WATCHDOG_STARTUP_GRACE}s heartbeat=${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}/${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr scripts=${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"
log "$ICON_WATCHDOG Order: $(for s in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do printf '%s ' "${s##*/}"; done)"

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"

# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
_watchdog_display_name() {
    local path="$1"
    local base="${path##*/}"
    base="${base%.sh}"
    base="${base//_/ }"
    echo "$base" | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2); print}'
}

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY WATCHDOG ORCHESTRATOR STATUS ━━━━━"
    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    echo ""

    UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
    if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
        warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)"
    else
        echo "Past startup grace — $(format_duration $UPTIME_S) uptime"
    fi

    echo ""
    echo "── Sub-scripts ──"
    for entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
        local_path="$ECOSYSTEM_ROOT/$entry"
        label="$(_watchdog_display_name "$entry")"
        if [[ -f "$local_path" ]]; then
            [[ -x "$local_path" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
            echo "  $icon $label — ${local_path##*/}"
        else
            echo "  $ICON_ERROR $label — NOT FOUND: $local_path"
        fi
    done

    echo ""
    echo "  Schedule:   */15 * * * * (every 15 minutes)"
    echo "  Heartbeat:  ${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true} / every ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Array Check ━━━
# ==============================================================================================
if ! platform_storage_healthy; then
    echo "Array not started — skipping watchdog cycle"
    exit 0
fi
log "$ICON_DISK Array: $(platform_storage_path) mounted ✅"

# ==============================================================================================
# ━━━ Startup Grace ━━━
# ==============================================================================================
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
    echo "Startup grace — $(format_duration $UPTIME_SECONDS) / $(format_duration $WATCHDOG_STARTUP_GRACE) — skipping cycle"
    exit 0
fi
log "Startup grace: past — uptime $(format_duration $UPTIME_SECONDS)"

# ==============================================================================================
# ━━━ Run Watchdog Cycle ━━━
# ==============================================================================================
CYCLE_START=$(date +%s)
PASS=()
FAIL=()

run_watchdog() {
    local name="$1" script="$2"

    if [[ ! -f "$script" ]]; then
        error "$name — not found: $script"
        FAIL+=("$name:missing")
        return 1
    fi

    [[ ! -x "$script" ]] && chmod +x "$script"

    local extra_args=()
    [[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
    [[ "$ENABLE_LOGGING" == true ]] && extra_args+=("--log")

    local _ws
    _ws=$(date +%s)
    log "$ICON_START $name"
    if bash "$script" "${extra_args[@]}"; then
        log "$ICON_DONE $name — done in $(format_duration $(( $(date +%s) - _ws )))"
        PASS+=("$name")
        return 0
    else
        error "$name — non-zero exit ($(format_duration $(( $(date +%s) - _ws ))))"
        FAIL+=("$name")
        return 1
    fi
}

for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
    run_watchdog "$(_watchdog_display_name "$_entry")" "$ECOSYSTEM_ROOT/$_entry"
done

CYCLE_END=$(date +%s)
DURATION=$(( CYCLE_END - CYCLE_START ))

# ==============================================================================================
# ━━━ Heartbeat ━━━
# ==============================================================================================
if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then
    HB_SECONDS=$(( ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1} * 3600 ))
    HB_COUNT_FILE="${STATE_DIR:-/tmp}/watchdog_orch_hb.count"
    HB_COUNT=$(cat "$HB_COUNT_FILE" 2>/dev/null || echo 0)
    HB_COUNT=$(( HB_COUNT + 1 ))
    echo "$HB_COUNT" > "$HB_COUNT_FILE"
    # Each cron run = ~60s — use count × 60 as uptime approximation
    HB_ELAPSED=$(( HB_COUNT * 60 ))
    if [[ "$HB_SECONDS" -gt 0 ]] && (( HB_ELAPSED % HB_SECONDS < 60 )) && [[ "$HB_COUNT" -gt 1 ]]; then
        HB_HR=$(( HB_ELAPSED / 3600 ))
        warn "♥ watchdog_orchestrator alive — $MY_ID — ~${HB_HR}hr ($(date '+%H:%M:%S'))"
    fi
fi

# ==============================================================================================
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
# ==============================================================================================
if [[ "${#FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
    for p in "${PASS[@]}"; do log "  $ICON_DONE  $p"; done
    for f in "${FAIL[@]}"; do error "  $ICON_ERROR $f"; done
    echo "$ICON_TIME  Duration: $(format_duration $DURATION)"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
else
    echo "$ICON_DONE Watchdog cycle — ${#PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
fi

if [[ "${#FAIL[@]}" -gt 0 ]]; then
    notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
        "Watchdog Orchestrator" "warning"
    exit 1
fi

exit 0