Add RAG retrieval core, and correct six stale default values in headers
Chunker splits on the header sections the audit standardised, then sub-splits named-paragraph safeguards — without that a specific question about one of rsync.sh's fourteen safeguards scored below unrelated chunks, because the other thirteen dominated the vector. Index is SQLite with raw float32 blobs and is incremental on mtime; a no-op re-index takes 66ms. The stale defaults were found by asking the system a question and checking its answer: it correctly reported what mover_stop.sh's header claimed, and the header was wrong.
This commit is contained in:
Gmer4Lfe
+176
@@ -0,0 +1,176 @@
|
||||
'use strict';
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// CLI bridge — the thin layer the bash entry points call.
|
||||
//
|
||||
// The bash scripts own configuration, gating, locking and logging, exactly as they do for every
|
||||
// other Varaverk job. This file owns only the work that is genuinely awkward in bash: float
|
||||
// vector math and SQLite BLOBs. That split follows the existing api_cache_writer.sh precedent —
|
||||
// a bash shim in front of the language that fits the task.
|
||||
//
|
||||
// Every value arrives as an argument or an environment variable read by the caller. This file
|
||||
// never reads a conf file itself, so there is exactly one place that decides what the settings
|
||||
// are.
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
const { buildIndex } = require('./index.js');
|
||||
const { search } = require('./search.js');
|
||||
|
||||
function arg(name, dflt) {
|
||||
const p = `--${name}=`;
|
||||
const hit = process.argv.find(a => a.startsWith(p));
|
||||
return hit ? hit.slice(p.length) : dflt;
|
||||
}
|
||||
function flag(name) {
|
||||
return process.argv.includes(`--${name}`);
|
||||
}
|
||||
|
||||
function fail(msg, code = 1) {
|
||||
console.error(msg);
|
||||
process.exit(code);
|
||||
}
|
||||
|
||||
async function cmdIndex() {
|
||||
const root = arg('root');
|
||||
const db = arg('db');
|
||||
const url = arg('url');
|
||||
const model = arg('model', 'nomic-embed-text');
|
||||
if (!root || !db || !url) fail('index: --root, --db and --url are required');
|
||||
|
||||
const quiet = flag('quiet');
|
||||
let stats;
|
||||
try {
|
||||
stats = await buildIndex({
|
||||
root, dbPath: db, url, model,
|
||||
batch: parseInt(arg('batch', '32'), 10),
|
||||
timeout: parseInt(arg('timeout', '120000'), 10),
|
||||
force: flag('force'),
|
||||
dryRun: flag('dry-run'),
|
||||
onProgress: p => {
|
||||
if (p.error) console.error(`embed batch failed: ${p.error}`);
|
||||
else if (!quiet && p.done % 320 === 0) console.log(` embedded ${p.done}/${p.total}`);
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
fail(`index failed: ${e.message}`, 2);
|
||||
}
|
||||
|
||||
if (flag('json')) { console.log(JSON.stringify(stats)); return; }
|
||||
if (stats.dryRun) {
|
||||
console.log(`DRY RUN — ${stats.files} file(s) would be indexed, ${stats.chunks} chunk(s) embedded`);
|
||||
console.log(` ${stats.skipped} unchanged, ${stats.removed} stale entr(ies) would be dropped`);
|
||||
return;
|
||||
}
|
||||
console.log(`indexed ${stats.files} file(s), ${stats.chunks} chunk(s) embedded`);
|
||||
console.log(` ${stats.skipped} unchanged, ${stats.removed} removed, ${stats.failed} failed`);
|
||||
console.log(` index now holds ${stats.total} chunk(s)`);
|
||||
// A partial index is usable but not complete — say so in the exit code so a caller can act.
|
||||
if (stats.failed) process.exit(3);
|
||||
}
|
||||
|
||||
async function cmdSearch() {
|
||||
const db = arg('db');
|
||||
const url = arg('url');
|
||||
const model = arg('model', 'nomic-embed-text');
|
||||
const q = arg('query');
|
||||
if (!db || !url || !q) fail('search: --db, --url and --query are required');
|
||||
|
||||
let r;
|
||||
try {
|
||||
r = await search({
|
||||
dbPath: db, url, embedModel: model, query: q,
|
||||
k: parseInt(arg('k', '8'), 10),
|
||||
perFile: parseInt(arg('per-file', '3'), 10),
|
||||
section: arg('section', null),
|
||||
kind: arg('kind', null),
|
||||
});
|
||||
} catch (e) {
|
||||
fail(`search failed: ${e.message}`, 2);
|
||||
}
|
||||
|
||||
if (flag('json')) { console.log(JSON.stringify(r)); return; }
|
||||
if (!r.results.length) { console.log('no matches'); return; }
|
||||
if (r.intents.length) console.log(`intent: ${r.intents.join(', ')}\n`);
|
||||
for (const x of r.results) {
|
||||
const label = x.heading || x.section || '-';
|
||||
console.log(`── ${x.score.toFixed(3)} ${x.path} [${x.section || x.kind}] ${label}`);
|
||||
console.log(x.content.split('\n').map(l => ' ' + l).join('\n'));
|
||||
console.log('');
|
||||
}
|
||||
}
|
||||
|
||||
// Retrieval + generation. The prompt is built here so the context block and the instructions
|
||||
// stay in one reviewable place.
|
||||
async function cmdAsk() {
|
||||
const db = arg('db');
|
||||
const url = arg('url');
|
||||
const embed = arg('embed-model', 'nomic-embed-text');
|
||||
const gen = arg('model');
|
||||
const q = arg('query');
|
||||
const timeout = parseInt(arg('timeout', '240000'), 10);
|
||||
if (!db || !url || !gen || !q) fail('ask: --db, --url, --model and --query are required');
|
||||
|
||||
let r;
|
||||
try {
|
||||
r = await search({
|
||||
dbPath: db, url, embedModel: embed, query: q,
|
||||
k: parseInt(arg('k', '6'), 10), perFile: parseInt(arg('per-file', '2'), 10),
|
||||
});
|
||||
} catch (e) {
|
||||
fail(`retrieval failed: ${e.message}`, 2);
|
||||
}
|
||||
if (!r.results.length) fail('no relevant context found in the index', 4);
|
||||
|
||||
const context = r.results.map((x, i) => {
|
||||
const label = [x.path, x.section, x.heading].filter(Boolean).join(' › ');
|
||||
return `[${i + 1}] ${label}\n${x.content}`;
|
||||
}).join('\n\n');
|
||||
|
||||
const prompt =
|
||||
`You are answering questions about Varaverk, a two-server self-healing home media ecosystem.
|
||||
|
||||
Answer ONLY from the context below. If the context does not contain the answer, say so plainly
|
||||
and name what is missing — do not fill the gap from general knowledge about Linux, Docker or
|
||||
rsync, because this system's conventions are frequently not the conventional ones.
|
||||
|
||||
Cite the source of each claim as [n]. Be concise and concrete.
|
||||
|
||||
CONTEXT
|
||||
${context}
|
||||
|
||||
QUESTION
|
||||
${q}
|
||||
|
||||
ANSWER`;
|
||||
|
||||
let res;
|
||||
try {
|
||||
res = await fetch(`${url.replace(/\/$/, '')}/api/generate`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: gen, prompt, stream: false,
|
||||
options: { temperature: 0.2, num_ctx: 8192 },
|
||||
}),
|
||||
signal: AbortSignal.timeout(timeout),
|
||||
});
|
||||
} catch (e) {
|
||||
fail(`generation failed: ${e.message}`, 2);
|
||||
}
|
||||
if (!res.ok) fail(`generation HTTP ${res.status}`, 2);
|
||||
const j = await res.json();
|
||||
|
||||
if (flag('json')) {
|
||||
console.log(JSON.stringify({ answer: j.response, sources: r.results.map(x => ({ path: x.path, section: x.section, heading: x.heading, score: x.score })) }));
|
||||
return;
|
||||
}
|
||||
console.log((j.response || '').trim());
|
||||
console.log('\nSources:');
|
||||
r.results.forEach((x, i) => {
|
||||
console.log(` [${i + 1}] ${[x.path, x.section, x.heading].filter(Boolean).join(' › ')}`);
|
||||
});
|
||||
}
|
||||
|
||||
const cmd = process.argv[2];
|
||||
const table = { index: cmdIndex, search: cmdSearch, ask: cmdAsk };
|
||||
if (!table[cmd]) fail(`usage: cli.js <index|search|ask> [--flags]`);
|
||||
table[cmd]().catch(e => fail(e.message, 2));
|
||||
Reference in New Issue
Block a user