#!/bin/bash
# ==============================================================================================
# ========================== AI Retrieval Index Builder ========================================
# ==============================================================================================
#
# PURPOSE
# ==============================================================================================
# Builds and refreshes the retrieval index over Varaverk's own documentation — every script
# header, folder README and Manual, and the conf templates. Chunks each file on the section
# boundaries the header convention already defines, embeds each chunk through Ollama, and
# stores the vectors in SQLite for AI/ai_query.sh to search.
#
# The index is derived data. It is gitignored, host-local, and rebuildable from the repo in
# minutes — nothing depends on it surviving.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
# Chunking mirrors the header convention rather than using a fixed token window:
#
#   bash headers      split on the six section names, then sub-split named-paragraph
#                     safeguards and principles so one question finds one answer
#   PHP headers       same six names plus the per-layer tails (EXPORTS, RENDERS, ...)
#   markdown          split on headings
#   conf templates    split on the ━━━ section rules
#
# Every chunk keeps its section name as a field, which is what lets a query about a safeguard
# be steered toward OPERATIONAL SAFEGUARDS chunks before similarity is considered.
#
# Incremental by file mtime. A file whose mtime has not moved is skipped without being read,
# so a routine refresh costs well under a second and a full rebuild costs a few minutes.
#
# Heavy lifting runs in Node — float vector maths and SQLite BLOBs are genuinely awkward in
# bash. This follows the api_cache_writer.sh precedent: a bash shim owning config, gating,
# locking and logging, in front of the language that fits the work.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Tracked Files Only
#   Indexes exactly what `git ls-files` reports. Configurations/, State_Files/ and data/ are
#   gitignored, so it is structurally impossible for a credential to reach the index — the
#   files holding them were never in the repo. This must never become a filesystem walk: a
#   secret written into a vector cannot be rotated back out of it.
#
# The Index Is Disposable
#   Stored under DATA_DIR, gitignored, and never synced to a partner. Losing it costs one
#   rebuild. Nothing reads it as a source of truth — it points at files, and the files are
#   the truth.
#
# Documentation Is The Corpus, Not The Code
#   Script bodies are not indexed. The headers state intent and the code states mechanism;
#   for the questions this answers, intent retrieves far better and is far cheaper.
#
# Off By Default
#   Does nothing unless AI_ENABLED is true. A node with AI off never pays for this.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Fail-Closed Gate
#   Exits cleanly unless AI_ENABLED is exactly "true". Any other value, including unset,
#   means off.
#
# Root Enforcement
#   Writes into DATA_DIR alongside other Varaverk state.
#
# Concurrency Lock
#   acquire_lock() prevents two indexers racing on the same database.
#
# Dependency Verification
#   Verifies node and the Ollama endpoint before touching the database. A missing dependency
#   is reported and exits non-zero rather than leaving a half-built index.
#
# Reachability Pre-flight
#   Probes the resolved Ollama URL with AI_CONNECT_TIMEOUT before starting. An unreachable
#   endpoint aborts immediately instead of failing once per batch across the whole corpus.
#
# Partial Failure Is Not Recorded As Success
#   A file whose chunks all failed to embed keeps its previous rows and its old mtime, so the
#   next run retries it. A run with any failed batch exits 3.
#
# Atomic Per-Run Write
#   All database changes commit in one transaction. An interrupted run leaves the previous
#   index intact rather than a partially rewritten one.
#
# Deleted Files Are Removed From The Index
#   A file that has left the repo has its chunks deleted, so retrieval cannot cite something
#   that no longer exists.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   AI_ENABLED
#       Master switch. Fail-closed — must be exactly "true".
#
#   AI_INDEX_DB
#       SQLite index path. (shipped default: $DATA_DIR/ai_index.db)
#
#   AI_INDEX_BATCH
#       Chunks per embed request. (shipped default: 32)
#
#   AI_CONNECT_TIMEOUT
#       Seconds for the reachability probe. (shipped default: 5)
#
#   AI_REQUEST_TIMEOUT
#       Seconds for a single embed batch. (shipped default: 240)
#
# host*.conf
#
#   HOST*_OLLAMA_URL
#       This host's Ollama endpoint. Empty means no local Ollama.
#
#   HOST*_OLLAMA_EMBED_MODEL
#       Embedding model. The generation model cannot embed.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# ai_index.sh
#     Incremental refresh — only files whose mtime moved are re-embedded.
#
# ai_index.sh --force
#     Full rebuild. Discards the existing index and re-embeds every chunk.
#
# ai_index.sh --dry-run
#     Report what would be indexed. Contacts nothing and writes nothing.
#
# ai_index.sh --status
#     Show index location, size, chunk counts by kind and section, and last build time.
#
# ai_index.sh --log
#     Verbose — per-batch embedding progress.
#
# ==============================================================================================

set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${SCRIPT_DIR}/../load_config.sh"

detect_hosts

DRY_RUN=false; FORCE=false; STATUS=false; LOG=false
for arg in "$@"; do
    case "$arg" in
        --dry-run) DRY_RUN=true ;;
        --force)   FORCE=true ;;
        --status)  STATUS=true ;;
        --log)     LOG=true ;;
        *) echo "Unknown option: $arg" >&2; exit 1 ;;
    esac
done

CLI="${SCRIPT_DIR}/lib/cli.js"
DB="${AI_INDEX_DB:-${DATA_DIR}/ai_index.db}"

_url_var="${MY_ID}_OLLAMA_URL"
_emb_var="${MY_ID}_OLLAMA_EMBED_MODEL"
OLLAMA_URL="${!_url_var:-}"
EMBED_MODEL="${!_emb_var:-nomic-embed-text}"

# ── Status ────────────────────────────────────────────────────────────────────────────────────
if [[ "$STATUS" == true ]]; then
    echo "$ICON_GEAR  AI Index Status"
    echo "   Enabled:   ${AI_ENABLED:-false}"
    echo "   Database:  $DB"
    if [[ -f "$DB" ]]; then
        echo "   Size:      $(du -h "$DB" 2>/dev/null | cut -f1)"
        echo "   Chunks:    $(sqlite3 "$DB" 'SELECT COUNT(*) FROM vv_chunks;' 2>/dev/null || echo '?')"
        echo "   Files:     $(sqlite3 "$DB" 'SELECT COUNT(*) FROM vv_files;'  2>/dev/null || echo '?')"
        _last=$(sqlite3 "$DB" "SELECT v FROM vv_meta WHERE k='last_index';" 2>/dev/null)
        [[ -n "$_last" ]] && echo "   Last built: $(date -d "@$_last" '+%Y-%m-%d %H:%M:%S' 2>/dev/null)"
        echo "   Model:     $(sqlite3 "$DB" "SELECT v FROM vv_meta WHERE k='embed_model';" 2>/dev/null || echo '?')"
        echo "   By kind:"
        sqlite3 "$DB" "SELECT '     '||kind||': '||COUNT(*) FROM vv_chunks GROUP BY kind ORDER BY COUNT(*) DESC;" 2>/dev/null
    else
        echo "   Database:  not built yet"
    fi
    echo "   Ollama:    ${OLLAMA_URL:-<none local>}"
    echo "   Embed:     $EMBED_MODEL"
    exit 0
fi

# ── Gate ──────────────────────────────────────────────────────────────────────────────────────
if [[ "${AI_ENABLED:-false}" != "true" ]]; then
    log "AI_ENABLED is not true — skipping index build"
    exit 0
fi

# The mesh shares one AI, and the index belongs to the node that holds the model. A mirror has the
# same checkout and could build one, but nothing there would read it: retrieval happens wherever
# generation happens, which is the owner.
#
# A skip, not an error. This is reached from git_pull_execute.sh on every node after every pull;
# before the AI became mesh-wide it ran here too and failed on the empty OLLAMA_URL, nightly and
# silently, because the caller discards its output.
_ai_owner="${AI_OWNER_HOST:-host1}"
if [[ "${MY_ID,,}" != "${_ai_owner,,}" ]]; then
    log "This node is not the AI owner ($_ai_owner) — the index lives there; skipping"
    exit 0
fi

if [[ "$DRY_RUN" == false && "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock

command -v node >/dev/null 2>&1 || { error "node not found — required to build the index"; exit 1; }
[[ -f "$CLI" ]] || { error "missing $CLI"; exit 1; }

# The AI owner has had data/ai since the subsystem was built, so nothing ever created it — cli.js
# opens the DB by path and does not make the directory. On a first build the failure surfaces as a
# sqlite open error rather than as the missing directory it is.
if [[ "$DRY_RUN" == false ]] && ! mkdir -p "$(dirname "$DB")"; then
    error "Cannot create $(dirname "$DB")"
    exit 1
fi

if [[ -z "$OLLAMA_URL" ]]; then
    error "${MY_ID}_OLLAMA_URL is empty — no local Ollama to index against"
    exit 1
fi

# Pre-flight: fail once, up front, rather than once per batch across the whole corpus.
if [[ "$DRY_RUN" == false ]]; then
    if ! curl -sf --max-time "${AI_CONNECT_TIMEOUT:-5}" "${OLLAMA_URL%/}/api/tags" >/dev/null 2>&1; then
        error "Ollama unreachable at $OLLAMA_URL"
        exit 1
    fi
fi

# ── Build ─────────────────────────────────────────────────────────────────────────────────────
_args=(
    index
    "--root=${SCRIPTS_DIR}"
    "--db=${DB}"
    "--url=${OLLAMA_URL}"
    "--model=${EMBED_MODEL}"
    "--batch=${AI_INDEX_BATCH:-32}"
    "--timeout=$(( ${AI_REQUEST_TIMEOUT:-240} * 1000 ))"
)
[[ "$FORCE"   == true ]] && _args+=(--force)
[[ "$DRY_RUN" == true ]] && _args+=(--dry-run)
[[ "$LOG"     == false ]] && _args+=(--quiet)

log "$ICON_GEAR Building AI index → $DB"
node --no-warnings "$CLI" "${_args[@]}"
_rc=$?

case "$_rc" in
    0) log "$ICON_DONE AI index build complete" ;;
    3) warn "AI index built with some batches failed — those files will retry next run" ;;
    *) error "AI index build failed (exit $_rc)" ;;
esac

exit "$_rc"