Varaverk/AI/lib/index.js

'use strict';
// ═══════════════════════════════════════════════════════════════════════════════════════════════
// Indexer — chunk the repo, embed each chunk, store vectors in SQLite.
//
// Incremental by file mtime: a file whose mtime has not moved since its last index is skipped
// entirely, so a routine re-index costs seconds rather than re-embedding the whole corpus.
//
// Vectors are stored as raw little-endian float32 BLOBs. nomic-embed-text returns L2-normalised
// vectors, so cosine similarity is a plain dot product at query time — no normalising, no
// magnitude cache. PHP can read the same blobs with unpack('f*', $blob) when the UI needs them.
// ═══════════════════════════════════════════════════════════════════════════════════════════════

const fs   = require('fs');
const path = require('path');
const { execSync } = require('child_process');
const { DatabaseSync } = require('node:sqlite');
const { chunkFile, classify } = require('./chunk.js');

const SCHEMA = `
CREATE TABLE IF NOT EXISTS vv_files (
    path     TEXT PRIMARY KEY,
    mtime    INTEGER NOT NULL,
    chunks   INTEGER NOT NULL,
    indexed  INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS vv_chunks (
    id       INTEGER PRIMARY KEY,
    path     TEXT NOT NULL,
    kind     TEXT NOT NULL,
    section  TEXT,
    heading  TEXT,
    part     INTEGER,
    content  TEXT NOT NULL,
    vector   BLOB NOT NULL,
    indexed  INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_chunks_path    ON vv_chunks(path);
CREATE INDEX IF NOT EXISTS idx_chunks_section ON vv_chunks(section);
CREATE INDEX IF NOT EXISTS idx_chunks_kind    ON vv_chunks(kind);
CREATE TABLE IF NOT EXISTS vv_meta (k TEXT PRIMARY KEY, v TEXT);
`;

function openDb(dbPath) {
    fs.mkdirSync(path.dirname(dbPath), { recursive: true });
    const db = new DatabaseSync(dbPath);
    db.exec('PRAGMA journal_mode = WAL;');
    db.exec('PRAGMA synchronous = NORMAL;');
    db.exec(SCHEMA);
    return db;
}

// Only ever index what git tracks. Configurations/, State_Files/ and data/ are gitignored, which
// is what makes it structurally impossible for a credential to reach the index — the files that
// hold them were never in the repo. Do not replace this with a filesystem walk.
function trackedFiles(root) {
    return execSync('git ls-files', { cwd: root, maxBuffer: 1 << 26 })
        .toString().trim().split('\n')
        .filter(Boolean)
        .filter(f => classify(f) !== 'other');
}

async function embedBatch(url, model, inputs, timeoutMs) {
    const ctl = AbortSignal.timeout(timeoutMs);
    const res = await fetch(`${url.replace(/\/$/, '')}/api/embed`, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ model, input: inputs }),
        signal: ctl,
    });
    if (!res.ok) throw new Error(`embed HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`);
    const j = await res.json();
    if (!j.embeddings || j.embeddings.length !== inputs.length)
        throw new Error(`embed returned ${j.embeddings ? j.embeddings.length : 0} of ${inputs.length}`);
    return j.embeddings;
}

function toBlob(vec) {
    return Buffer.from(Float32Array.from(vec).buffer);
}

async function buildIndex(opts) {
    const {
        root, dbPath, url, model,
        batch = 32, timeout = 120000, force = false, dryRun = false,
        onProgress = () => {},
    } = opts;

    const db  = dryRun ? null : openDb(dbPath);
    const now = Math.floor(Date.now() / 1000);

    const known = new Map();
    if (db) for (const r of db.prepare('SELECT path, mtime FROM vv_files').all()) known.set(r.path, r.mtime);

    const files   = trackedFiles(root);
    const present = new Set(files);

    // Files that left the repo must leave the index with them.
    let removed = 0;
    if (db && !force) {
        for (const p of known.keys()) {
            if (!present.has(p)) {
                db.prepare('DELETE FROM vv_chunks WHERE path = ?').run(p);
                db.prepare('DELETE FROM vv_files  WHERE path = ?').run(p);
                removed++;
            }
        }
    }
    if (db && force) { db.exec('DELETE FROM vv_chunks; DELETE FROM vv_files;'); }

    // ── Collect the chunks that actually need embedding ───────────────────────────────────────
    const pending = [];
    let skipped = 0, scanned = 0;

    for (const rel of files) {
        const abs = path.join(root, rel);
        let st;
        try { st = fs.statSync(abs); } catch { continue; }
        const mtime = Math.floor(st.mtimeMs / 1000);
        scanned++;

        if (!force && known.has(rel) && known.get(rel) === mtime) { skipped++; continue; }

        let chunks = [];
        try { chunks = chunkFile(abs, rel); } catch (e) { continue; }
        // A file that yields no chunks still gets a vv_files row so it is not re-chunked every
        // run. Queued rather than written here, so every database change lands in the single
        // commit below — a run interrupted mid-embed must leave the index exactly as it was.
        pending.push({ rel, mtime, chunks });
    }

    const totalChunks = pending.reduce((n, f) => n + f.chunks.length, 0);
    if (dryRun) {
        return { dryRun: true, scanned, skipped, removed, files: pending.length, chunks: totalChunks };
    }

    // ── Embed in batches, write per file so an interrupted run leaves a consistent index ───────
    const flat = [];
    for (const f of pending) for (const c of f.chunks) flat.push({ f, c });

    let done = 0, failed = 0;
    for (let i = 0; i < flat.length; i += batch) {
        const slice  = flat.slice(i, i + batch);
        const inputs = slice.map(x => x.c.content);
        let vecs;
        try {
            vecs = await embedBatch(url, model, inputs, timeout);
        } catch (e) {
            failed += slice.length;
            onProgress({ done, total: flat.length, error: e.message });
            continue;
        }
        slice.forEach((x, k) => { x.c.__vec = vecs[k]; });
        done += slice.length;
        onProgress({ done, total: flat.length });
    }

    const ins = db.prepare(
        'INSERT INTO vv_chunks (path,kind,section,heading,part,content,vector,indexed) VALUES (?,?,?,?,?,?,?,?)'
    );
    const insF = db.prepare('INSERT OR REPLACE INTO vv_files VALUES (?,?,?,?)');

    db.exec('BEGIN');
    try {
        for (const f of pending) {
            // Nothing to index in this file at all — record it so it is not re-chunked next run.
            if (!f.chunks.length) {
                db.prepare('DELETE FROM vv_chunks WHERE path = ?').run(f.rel);
                insF.run(f.rel, f.mtime, 0, now);
                continue;
            }
            const embedded = f.chunks.filter(c => c.__vec);
            // A file whose chunks all failed to embed keeps its previous rows and its old mtime,
            // so the next run retries it rather than recording a half-indexed file as current.
            if (!embedded.length) continue;
            db.prepare('DELETE FROM vv_chunks WHERE path = ?').run(f.rel);
            for (const c of embedded) {
                ins.run(c.path, c.kind, c.section, c.heading, c.part, c.content, toBlob(c.__vec), now);
            }
            insF.run(f.rel, f.mtime, embedded.length, now);
        }
        db.prepare('INSERT OR REPLACE INTO vv_meta VALUES (?,?)').run('embed_model', model);
        db.prepare('INSERT OR REPLACE INTO vv_meta VALUES (?,?)').run('last_index', String(now));
        db.prepare('INSERT OR REPLACE INTO vv_meta VALUES (?,?)').run('dims', '768');
        db.exec('COMMIT');
    } catch (e) {
        db.exec('ROLLBACK');
        throw e;
    }

    const stats = {
        scanned, skipped, removed,
        files:  pending.length,
        chunks: done,
        failed,
        total:  db.prepare('SELECT COUNT(*) n FROM vv_chunks').get().n,
    };
    db.close();
    return stats;
}

module.exports = { buildIndex, openDb, toBlob, trackedFiles };