Add RAG retrieval core, and correct six stale default values in headers
Chunker splits on the header sections the audit standardised, then sub-splits named-paragraph safeguards — without that a specific question about one of rsync.sh's fourteen safeguards scored below unrelated chunks, because the other thirteen dominated the vector. Index is SQLite with raw float32 blobs and is incremental on mtime; a no-op re-index takes 66ms. The stale defaults were found by asking the system a question and checking its answer: it correctly reported what mover_stop.sh's header claimed, and the header was wrong.
This commit is contained in:
@@ -0,0 +1,121 @@
|
||||
'use strict';
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// Search — embed a question, score it against the index, return the best chunks.
|
||||
//
|
||||
// nomic-embed-text returns L2-normalised vectors, so cosine similarity is a plain dot product.
|
||||
// At this corpus size (~1.7k chunks x 768 dims) that is a couple of million multiply-adds —
|
||||
// under a millisecond, with no vector database and no index structure to maintain.
|
||||
//
|
||||
// Section routing is the payoff from the header audit. Every chunk knows whether it is a
|
||||
// PURPOSE, a DESIGN PRINCIPLES, an OPERATIONAL SAFEGUARDS and so on, so a question's shape can
|
||||
// steer retrieval before similarity is even considered. It is applied as a score boost rather
|
||||
// than a hard filter — intent detection is a heuristic, and a heuristic should not be able to
|
||||
// exclude the one chunk that actually holds the answer.
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
|
||||
// Question shape → the section most likely to answer it.
|
||||
const INTENTS = [
|
||||
{ section: 'OPERATIONAL SAFEGUARDS',
|
||||
re: /\b(safe|safety|guard|protect|prevent|fail|failure|abort|refuse|lock|root|timeout|dry.?run|what stops|what happens if|race|corrupt|data.?loss)\b/i },
|
||||
{ section: 'CONFIGURATION',
|
||||
re: /\b(variable|var|setting|conf|config|threshold|toggle|which key|what controls|where is .* set|default value|env)\b/i },
|
||||
{ section: 'RUNTIME MODES',
|
||||
re: /\b(flag|argument|option|--\w+|how do i run|invoke|cli|command line|status mode|usage)\b/i },
|
||||
{ section: 'DESIGN PRINCIPLES',
|
||||
re: /\b(why|rationale|reason|design|decision|deliberate|intentional|on purpose|trade.?off|chose|approach)\b/i },
|
||||
{ section: 'OPERATIONAL MODEL',
|
||||
re: /\b(how does .* work|flow|sequence|order|tier|lifecycle|state machine|when does)\b/i },
|
||||
{ section: 'EXPORTS',
|
||||
re: /\b(function|export|api surface|what does .* provide|helper|vv_\w+)\b/i },
|
||||
{ section: 'PURPOSE',
|
||||
re: /\b(what is|what does .* do|purpose|responsible for|job of)\b/i },
|
||||
];
|
||||
|
||||
const SECTION_BOOST = 0.06; // enough to reorder near-ties, not enough to beat a real match
|
||||
const KIND_BOOST = 0.02; // docs answer "how do I" better than a script header does
|
||||
|
||||
function detectIntent(q) {
|
||||
const hits = [];
|
||||
for (const i of INTENTS) if (i.re.test(q)) hits.push(i.section);
|
||||
return hits;
|
||||
}
|
||||
|
||||
function blobToVec(buf) {
|
||||
const b = Buffer.from(buf);
|
||||
return new Float32Array(b.buffer, b.byteOffset, b.length / 4);
|
||||
}
|
||||
|
||||
function dot(a, b) {
|
||||
let s = 0;
|
||||
for (let i = 0; i < a.length; i++) s += a[i] * b[i];
|
||||
return s;
|
||||
}
|
||||
|
||||
async function embedQuery(url, model, text, timeoutMs = 60000) {
|
||||
const res = await fetch(`${url.replace(/\/$/, '')}/api/embed`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ model, input: text }),
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
if (!res.ok) throw new Error(`embed HTTP ${res.status}`);
|
||||
const j = await res.json();
|
||||
if (!j.embeddings || !j.embeddings[0]) throw new Error('embed returned no vector');
|
||||
return Float32Array.from(j.embeddings[0]);
|
||||
}
|
||||
|
||||
// Keep at most `perFile` chunks from any one file, so a single large document cannot fill the
|
||||
// entire context window and crowd out a better answer living somewhere else.
|
||||
function diversify(rows, k, perFile) {
|
||||
const seen = new Map();
|
||||
const out = [];
|
||||
for (const r of rows) {
|
||||
const n = seen.get(r.path) || 0;
|
||||
if (n >= perFile) continue;
|
||||
seen.set(r.path, n + 1);
|
||||
out.push(r);
|
||||
if (out.length >= k) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function search(opts) {
|
||||
const {
|
||||
dbPath, url, embedModel, query,
|
||||
k = 8, perFile = 3, section = null, kind = null, minScore = 0.0,
|
||||
} = opts;
|
||||
|
||||
const db = new DatabaseSync(dbPath, { readOnly: true });
|
||||
|
||||
let sql = 'SELECT id,path,kind,section,heading,part,content,vector FROM vv_chunks';
|
||||
const where = [], args = [];
|
||||
if (section) { where.push('section = ?'); args.push(section); }
|
||||
if (kind) { where.push('kind = ?'); args.push(kind); }
|
||||
if (where.length) sql += ' WHERE ' + where.join(' AND ');
|
||||
|
||||
const rows = db.prepare(sql).all(...args);
|
||||
if (!rows.length) { db.close(); return { results: [], intents: [], scanned: 0 }; }
|
||||
|
||||
const qv = await embedQuery(url, embedModel, query);
|
||||
const intents = section ? [] : detectIntent(query);
|
||||
const wantDoc = /\b(how do i|steps|procedure|setup|install|troubleshoot|guide)\b/i.test(query);
|
||||
|
||||
const scored = rows.map(r => {
|
||||
let s = dot(qv, blobToVec(r.vector));
|
||||
if (intents.includes(r.section)) s += SECTION_BOOST;
|
||||
if (wantDoc && (r.kind === 'manual' || r.kind === 'readme')) s += KIND_BOOST;
|
||||
return {
|
||||
id: r.id, path: r.path, kind: r.kind, section: r.section,
|
||||
heading: r.heading, part: r.part, content: r.content, score: s,
|
||||
};
|
||||
});
|
||||
|
||||
scored.sort((a, b) => b.score - a.score);
|
||||
const kept = diversify(scored.filter(r => r.score >= minScore), k, perFile);
|
||||
db.close();
|
||||
return { results: kept, intents, scanned: rows.length };
|
||||
}
|
||||
|
||||
module.exports = { search, detectIntent, blobToVec, dot, embedQuery, INTENTS };
|
||||
Reference in New Issue
Block a user