Varaverk/AI/lib/search.js

'use strict';
// ═══════════════════════════════════════════════════════════════════════════════════════════════
// Search — embed a question, score it against the index, return the best chunks.
//
// nomic-embed-text returns L2-normalised vectors, so cosine similarity is a plain dot product.
// At this corpus size (~1.7k chunks x 768 dims) that is a couple of million multiply-adds —
// under a millisecond, with no vector database and no index structure to maintain.
//
// Section routing is the payoff from the header audit. Every chunk knows whether it is a
// PURPOSE, a DESIGN PRINCIPLES, an OPERATIONAL SAFEGUARDS and so on, so a question's shape can
// steer retrieval before similarity is even considered. It is applied as a score boost rather
// than a hard filter — intent detection is a heuristic, and a heuristic should not be able to
// exclude the one chunk that actually holds the answer.
// ═══════════════════════════════════════════════════════════════════════════════════════════════

const { DatabaseSync } = require('node:sqlite');

// Question shape → the section most likely to answer it.
const INTENTS = [
    { section: 'OPERATIONAL SAFEGUARDS',
      re: /\b(safe|safety|guard|protect|prevent|fail|failure|abort|refuse|lock|root|timeout|dry.?run|what stops|what happens if|race|corrupt|data.?loss)\b/i },
    { section: 'CONFIGURATION',
      re: /\b(variable|var|setting|conf|config|threshold|toggle|which key|what controls|where is .* set|default value|env)\b/i },
    { section: 'RUNTIME MODES',
      re: /\b(flag|argument|option|--\w+|how do i run|invoke|cli|command line|status mode|usage)\b/i },
    { section: 'DESIGN PRINCIPLES',
      re: /\b(why|rationale|reason|design|decision|deliberate|intentional|on purpose|trade.?off|chose|approach)\b/i },
    { section: 'OPERATIONAL MODEL',
      re: /\b(how does .* work|flow|sequence|order|tier|lifecycle|state machine|when does)\b/i },
    { section: 'EXPORTS',
      re: /\b(function|export|api surface|what does .* provide|helper|vv_\w+)\b/i },
    { section: 'PURPOSE',
      re: /\b(what is|what does .* do|purpose|responsible for|job of)\b/i },
];

const SECTION_BOOST = 0.06;   // enough to reorder near-ties, not enough to beat a real match
const KIND_BOOST    = 0.02;   // docs answer "how do I" better than a script header does

function detectIntent(q) {
    const hits = [];
    for (const i of INTENTS) if (i.re.test(q)) hits.push(i.section);
    return hits;
}

function blobToVec(buf) {
    const b = Buffer.from(buf);
    return new Float32Array(b.buffer, b.byteOffset, b.length / 4);
}

function dot(a, b) {
    let s = 0;
    for (let i = 0; i < a.length; i++) s += a[i] * b[i];
    return s;
}

async function embedQuery(url, model, text, timeoutMs = 60000) {
    const res = await fetch(`${url.replace(/\/$/, '')}/api/embed`, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ model, input: text }),
        signal: AbortSignal.timeout(timeoutMs),
    });
    if (!res.ok) throw new Error(`embed HTTP ${res.status}`);
    const j = await res.json();
    if (!j.embeddings || !j.embeddings[0]) throw new Error('embed returned no vector');
    return Float32Array.from(j.embeddings[0]);
}

// Keep at most `perFile` chunks from any one file, so a single large document cannot fill the
// entire context window and crowd out a better answer living somewhere else.
function diversify(rows, k, perFile) {
    const seen = new Map();
    const out  = [];
    for (const r of rows) {
        const n = seen.get(r.path) || 0;
        if (n >= perFile) continue;
        seen.set(r.path, n + 1);
        out.push(r);
        if (out.length >= k) break;
    }
    return out;
}

async function search(opts) {
    const {
        dbPath, url, embedModel, query,
        k = 8, perFile = 3, section = null, kind = null, minScore = 0.0,
    } = opts;

    const db = new DatabaseSync(dbPath, { readOnly: true });

    let sql = 'SELECT id,path,kind,section,heading,part,content,vector FROM vv_chunks';
    const where = [], args = [];
    if (section) { where.push('section = ?'); args.push(section); }
    if (kind)    { where.push('kind = ?');    args.push(kind); }
    if (where.length) sql += ' WHERE ' + where.join(' AND ');

    const rows = db.prepare(sql).all(...args);
    if (!rows.length) { db.close(); return { results: [], intents: [], scanned: 0 }; }

    const qv  = await embedQuery(url, embedModel, query);
    let intents = section ? [] : detectIntent(query);

    // "What is Varaverk" and "what is arr_sync.sh" are not the same question, and the PURPOSE
    // intent cannot tell them apart — it fires on both and boosts every PURPOSE block in the
    // repository at once. There are a couple of hundred, each genuinely describing the purpose of
    // something, and each a short sentence containing the word Varaverk. The project's own README
    // then loses to a script that migrates storage modes, because a paragraph is more diluted
    // than a one-line summary.
    //
    // A question that names the project and no component inside it is asking about the whole, so
    // PURPOSE is precisely the wrong section to promote. Dropping only that intent, rather than
    // all of them, leaves "why was Varaverk built this way" still routed to DESIGN PRINCIPLES.
    const namesProject   = /\bvaraverk\b/i.test(query);
    const namesComponent = /\b[\w.-]+\.(sh|php|js)\b|\b[A-Z][A-Z0-9]*(_[A-Z0-9]+)+\b/.test(query);
    const projectLevel   = namesProject && !namesComponent;
    if (projectLevel) intents = intents.filter(s => s !== 'PURPOSE');

    // The same question wants the top-level prose, which is what the doc kinds are.
    const wantDoc = projectLevel
                 || /\b(how do i|steps|procedure|setup|install|troubleshoot|guide)\b/i.test(query);

    const scored = rows.map(r => {
        let s = dot(qv, blobToVec(r.vector));
        if (intents.includes(r.section)) s += SECTION_BOOST;
        if (wantDoc && (r.kind === 'manual' || r.kind === 'readme')) s += KIND_BOOST;
        return {
            id: r.id, path: r.path, kind: r.kind, section: r.section,
            heading: r.heading, part: r.part, content: r.content, score: s,
        };
    });

    scored.sort((a, b) => b.score - a.score);
    const kept = diversify(scored.filter(r => r.score >= minScore), k, perFile);
    db.close();
    return { results: kept, intents, scanned: rows.length };
}

module.exports = { search, detectIntent, blobToVec, dot, embedQuery, INTENTS };