Add RAG retrieval core, and correct six stale default values in headers

Chunker splits on the header sections the audit standardised, then sub-splits
named-paragraph safeguards — without that a specific question about one of
rsync.sh's fourteen safeguards scored below unrelated chunks, because the other
thirteen dominated the vector. Index is SQLite with raw float32 blobs and is
incremental on mtime; a no-op re-index takes 66ms.

The stale defaults were found by asking the system a question and checking its
answer: it correctly reported what mover_stop.sh's header claimed, and the
header was wrong.
This commit is contained in:
Gmer4Lfe
2026-08-02 01:13:01 -04:00
parent ccc6c742ee
commit b00688bad1
9 changed files with 761 additions and 6 deletions
+121
View File
@@ -0,0 +1,121 @@
'use strict';
// ═══════════════════════════════════════════════════════════════════════════════════════════════
// Search — embed a question, score it against the index, return the best chunks.
//
// nomic-embed-text returns L2-normalised vectors, so cosine similarity is a plain dot product.
// At this corpus size (~1.7k chunks x 768 dims) that is a couple of million multiply-adds —
// under a millisecond, with no vector database and no index structure to maintain.
//
// Section routing is the payoff from the header audit. Every chunk knows whether it is a
// PURPOSE, a DESIGN PRINCIPLES, an OPERATIONAL SAFEGUARDS and so on, so a question's shape can
// steer retrieval before similarity is even considered. It is applied as a score boost rather
// than a hard filter — intent detection is a heuristic, and a heuristic should not be able to
// exclude the one chunk that actually holds the answer.
// ═══════════════════════════════════════════════════════════════════════════════════════════════
const { DatabaseSync } = require('node:sqlite');
// Question shape → the section most likely to answer it.
const INTENTS = [
{ section: 'OPERATIONAL SAFEGUARDS',
re: /\b(safe|safety|guard|protect|prevent|fail|failure|abort|refuse|lock|root|timeout|dry.?run|what stops|what happens if|race|corrupt|data.?loss)\b/i },
{ section: 'CONFIGURATION',
re: /\b(variable|var|setting|conf|config|threshold|toggle|which key|what controls|where is .* set|default value|env)\b/i },
{ section: 'RUNTIME MODES',
re: /\b(flag|argument|option|--\w+|how do i run|invoke|cli|command line|status mode|usage)\b/i },
{ section: 'DESIGN PRINCIPLES',
re: /\b(why|rationale|reason|design|decision|deliberate|intentional|on purpose|trade.?off|chose|approach)\b/i },
{ section: 'OPERATIONAL MODEL',
re: /\b(how does .* work|flow|sequence|order|tier|lifecycle|state machine|when does)\b/i },
{ section: 'EXPORTS',
re: /\b(function|export|api surface|what does .* provide|helper|vv_\w+)\b/i },
{ section: 'PURPOSE',
re: /\b(what is|what does .* do|purpose|responsible for|job of)\b/i },
];
const SECTION_BOOST = 0.06; // enough to reorder near-ties, not enough to beat a real match
const KIND_BOOST = 0.02; // docs answer "how do I" better than a script header does
function detectIntent(q) {
const hits = [];
for (const i of INTENTS) if (i.re.test(q)) hits.push(i.section);
return hits;
}
function blobToVec(buf) {
const b = Buffer.from(buf);
return new Float32Array(b.buffer, b.byteOffset, b.length / 4);
}
function dot(a, b) {
let s = 0;
for (let i = 0; i < a.length; i++) s += a[i] * b[i];
return s;
}
async function embedQuery(url, model, text, timeoutMs = 60000) {
const res = await fetch(`${url.replace(/\/$/, '')}/api/embed`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ model, input: text }),
signal: AbortSignal.timeout(timeoutMs),
});
if (!res.ok) throw new Error(`embed HTTP ${res.status}`);
const j = await res.json();
if (!j.embeddings || !j.embeddings[0]) throw new Error('embed returned no vector');
return Float32Array.from(j.embeddings[0]);
}
// Keep at most `perFile` chunks from any one file, so a single large document cannot fill the
// entire context window and crowd out a better answer living somewhere else.
function diversify(rows, k, perFile) {
const seen = new Map();
const out = [];
for (const r of rows) {
const n = seen.get(r.path) || 0;
if (n >= perFile) continue;
seen.set(r.path, n + 1);
out.push(r);
if (out.length >= k) break;
}
return out;
}
async function search(opts) {
const {
dbPath, url, embedModel, query,
k = 8, perFile = 3, section = null, kind = null, minScore = 0.0,
} = opts;
const db = new DatabaseSync(dbPath, { readOnly: true });
let sql = 'SELECT id,path,kind,section,heading,part,content,vector FROM vv_chunks';
const where = [], args = [];
if (section) { where.push('section = ?'); args.push(section); }
if (kind) { where.push('kind = ?'); args.push(kind); }
if (where.length) sql += ' WHERE ' + where.join(' AND ');
const rows = db.prepare(sql).all(...args);
if (!rows.length) { db.close(); return { results: [], intents: [], scanned: 0 }; }
const qv = await embedQuery(url, embedModel, query);
const intents = section ? [] : detectIntent(query);
const wantDoc = /\b(how do i|steps|procedure|setup|install|troubleshoot|guide)\b/i.test(query);
const scored = rows.map(r => {
let s = dot(qv, blobToVec(r.vector));
if (intents.includes(r.section)) s += SECTION_BOOST;
if (wantDoc && (r.kind === 'manual' || r.kind === 'readme')) s += KIND_BOOST;
return {
id: r.id, path: r.path, kind: r.kind, section: r.section,
heading: r.heading, part: r.part, content: r.content, score: s,
};
});
scored.sort((a, b) => b.score - a.score);
const kept = diversify(scored.filter(r => r.score >= minScore), k, perFile);
db.close();
return { results: kept, intents, scanned: rows.length };
}
module.exports = { search, detectIntent, blobToVec, dot, embedQuery, INTENTS };