diff --git a/AI/.gitignore b/AI/.gitignore new file mode 100644 index 0000000..936a447 --- /dev/null +++ b/AI/.gitignore @@ -0,0 +1,4 @@ +# The index is generated, host-specific, and regenerable in minutes. Never commit it. +*.db +*.db-wal +*.db-shm diff --git a/AI/lib/chunk.js b/AI/lib/chunk.js new file mode 100644 index 0000000..0de7aa1 --- /dev/null +++ b/AI/lib/chunk.js @@ -0,0 +1,255 @@ +'use strict'; +// ═══════════════════════════════════════════════════════════════════════════════════════════════ +// Chunker — turns repo files into retrieval units. +// +// The whole point of the header audit is that chunk boundaries are deterministic here. Bash +// scripts split on their six section names, markdown on its headings, conf templates on their +// ━━━ section rules. Nothing is split on a fixed token window, so no chunk ever contains half +// of one idea and half of another. +// +// Every chunk carries its section name as its own field, because that is the metadata that +// lets retrieval filter by question shape before it ever computes similarity. +// ═══════════════════════════════════════════════════════════════════════════════════════════════ + +const fs = require('fs'); +const path = require('path'); + +const BASH_SECTIONS = [ + 'PURPOSE', 'OPERATIONAL MODEL', 'DESIGN PRINCIPLES', + 'OPERATIONAL SAFEGUARDS', 'CONFIGURATION', 'RUNTIME MODES', +]; + +// PHP headers reuse the first three names deliberately, then diverge per layer. +const PHP_SECTIONS = [ + 'PURPOSE', 'OPERATIONAL MODEL', 'DESIGN PRINCIPLES', 'OPERATIONAL SAFEGUARDS', + 'STATUS', 'EXPORTS', 'REQUEST CONTRACT', 'SIDE EFFECTS', 'RENDERS', 'DEPENDS ON', + 'CONFIGURATION', +]; + +const MIN_CHARS = 40; // below this a chunk carries no retrievable meaning +const MAX_CHARS = 6000; // above this, split on blank lines — protects the embed window + +// Banner rules and box-drawing art are everywhere in this repo's headers. They carry no +// meaning to embed, and a chunk that is mostly rule characters is pure noise in the index. +// Measure a chunk by what is left after the decoration is removed, not by raw length. +function meaningful(s) { + return s.replace(/[═─━=_#\/*\s|+.-]/g, '').length; +} +const MIN_MEANINGFUL = 30; + +function isRealHeading(h) { + return !!h && /[A-Za-z0-9]/.test(h.replace(/[═─━=_]/g, '')); +} + +function stripPrefix(line, prefix) { + // '# text' -> 'text' '// text' -> 'text' + const re = new RegExp('^\\s*' + prefix + '\\s?'); + return line.replace(re, ''); +} + +// ── Comment-header sectioning, shared by bash (#) and PHP (//) ──────────────────────────────── +function sectionsFromCommentHeader(text, prefix, names) { + const lines = text.split('\n'); + const nameSet = new Set(names); + const found = []; + + // headerEnd matters as much as the section starts. The last section (RUNTIME MODES in bash, + // DEPENDS ON in a page) would otherwise run to EOF and sweep up every unrelated comment in + // the file — scheduler.php alone contributed an 11k-char chunk of unrelated inline comments. + let headerEnd = lines.length; + for (let i = 0; i < lines.length; i++) { + const raw = lines[i]; + if (!new RegExp('^\\s*' + prefix).test(raw)) { + // Header block ends at the first non-comment, non-blank line past the shebang. + // '' bracket a PHP header block and are not the end of it. + const t = raw.trim(); + if (t !== '' && !/^#!/.test(t) && t !== '' && found.length) { + headerEnd = i; + break; + } + continue; + } + const inner = stripPrefix(raw, prefix).trim(); + if (nameSet.has(inner)) found.push({ name: inner, start: i }); + } + + const out = []; + for (let k = 0; k < found.length; k++) { + const start = found[k].start + 1; + const end = k + 1 < found.length ? found[k + 1].start : headerEnd; + const body = lines.slice(start, end) + .filter(l => new RegExp('^\\s*' + prefix).test(l)) + .map(l => stripPrefix(l, prefix)) + // drop pure separator rules (════, ────, ━━━) — they carry no meaning + .filter(l => !/^[\s═─━=_-]*$/.test(l) || l.trim() === '') + .join('\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); + if (meaningful(body) >= MIN_MEANINGFUL) { + for (const p of splitNamedParagraphs(body)) + out.push({ section: found[k].name, title: p.title, content: p.content }); + } + } + return out; +} + +// ── Named-paragraph sub-chunking ────────────────────────────────────────────────────────────── +// The header convention writes safeguards and principles as named paragraphs: an unindented +// title line followed by an indented body. Embedding a whole section as one unit dilutes them — +// rsync.sh's OPERATIONAL SAFEGUARDS holds eight distinct guarantees in 2.8k chars, and a query +// about one of them scored below unrelated chunks because the other seven dominated the vector. +// Splitting on the title lines is what makes a specific question find a specific answer. +// +// The section name is carried onto every sub-chunk, so section routing still works; the +// paragraph title becomes the chunk's heading. +function splitNamedParagraphs(body) { + const lines = body.split('\n'); + const marks = []; + for (let i = 0; i < lines.length; i++) { + const l = lines[i]; + if (!l.trim()) continue; + if (/^\s/.test(l)) continue; // indented => body, not a title + if (/^[-*•]/.test(l.trim())) continue; // list item, not a title + if (l.trim().length > 80) continue; // a long line is prose, not a heading + if (/[.:;,]$/.test(l.trim())) continue; // ends like a sentence + + // The decisive signal: a real title is followed by an indented body. Wrapped prose is + // followed by more unindented prose. Without this check, any short line in a paragraph + // that happened to wrap became a spurious chunk boundary mid-sentence. + let j = i + 1; + while (j < lines.length && !lines[j].trim()) j++; + if (j >= lines.length || !/^\s+\S/.test(lines[j])) continue; + + marks.push(i); + } + // Fewer than two titles means this section is not written as named paragraphs — keep it whole. + if (marks.length < 2) return [{ title: null, content: body }]; + + const out = []; + if (marks[0] > 0) { + const pre = lines.slice(0, marks[0]).join('\n').trim(); + if (meaningful(pre) >= MIN_MEANINGFUL) out.push({ title: null, content: pre }); + } + for (let k = 0; k < marks.length; k++) { + const start = marks[k]; + const end = k + 1 < marks.length ? marks[k + 1] : lines.length; + const title = lines[start].trim(); + const content = lines.slice(start, end).join('\n').trim(); + if (meaningful(content) >= MIN_MEANINGFUL) out.push({ title, content }); + } + return out; +} + +// ── Markdown: split on ## headings, keep the heading with its body ───────────────────────────── +function sectionsFromMarkdown(text) { + const lines = text.split('\n'); + const marks = []; + let fence = false; + + for (let i = 0; i < lines.length; i++) { + if (/^\s*```/.test(lines[i])) { fence = !fence; continue; } + if (fence) continue; + if (/^#{1,3}\s+\S/.test(lines[i])) marks.push(i); + } + if (!marks.length) return [{ heading: null, content: text.trim() }]; + + const out = []; + // preamble before the first heading + if (marks[0] > 0) { + const pre = lines.slice(0, marks[0]).join('\n').trim(); + if (meaningful(pre) >= MIN_MEANINGFUL) out.push({ heading: null, content: pre }); + } + for (let k = 0; k < marks.length; k++) { + const start = marks[k]; + const end = k + 1 < marks.length ? marks[k + 1] : lines.length; + const heading = lines[start].replace(/^#+\s*/, '').replace(/[━─═]+/g, '').trim(); + const content = lines.slice(start, end).join('\n').trim(); + if (meaningful(content) >= MIN_MEANINGFUL) out.push({ heading: isRealHeading(heading) ? heading : null, content }); + } + return out; +} + +// ── Conf templates: split on the ━━━ / ── section rules ─────────────────────────────────────── +function sectionsFromConfTemplate(text) { + const lines = text.split('\n'); + const marks = []; + for (let i = 0; i < lines.length; i++) { + const m = lines[i].match(/^#\s*[━─]{2,}\s*(.+?)\s*[━─]{2,}\s*$/); + if (m && isRealHeading(m[1])) marks.push({ i, name: m[1].trim() }); + } + if (!marks.length) return []; + + const out = []; + for (let k = 0; k < marks.length; k++) { + const start = marks[k].i; + const end = k + 1 < marks.length ? marks[k + 1].i : lines.length; + const content = lines.slice(start, end).join('\n').replace(/\n{3,}/g, '\n\n').trim(); + if (meaningful(content) >= MIN_MEANINGFUL) out.push({ heading: marks[k].name, content }); + } + return out; +} + +// Oversized chunks split on blank lines rather than mid-sentence. +function capSize(chunks) { + const out = []; + for (const c of chunks) { + if (c.content.length <= MAX_CHARS) { out.push(c); continue; } + const paras = c.content.split(/\n\s*\n/); + let buf = [], len = 0, part = 1; + const flush = () => { + if (!buf.length) return; + out.push({ ...c, content: buf.join('\n\n'), part: part++ }); + buf = []; len = 0; + }; + for (const p of paras) { + if (len + p.length > MAX_CHARS && buf.length) flush(); + buf.push(p); len += p.length + 2; + } + flush(); + } + return out; +} + +function classify(rel) { + const base = path.basename(rel); + if (rel.startsWith('Deployment/') && rel.endsWith('.template')) return 'template'; + if (base.endsWith('.md')) { + if (base.startsWith('Manual')) return 'manual'; + if (base.startsWith('README') || base === 'README.md') return 'readme'; + return 'doc'; + } + if (base.endsWith('.sh')) return 'header'; + if (base.endsWith('.php')) return 'header'; + return 'other'; +} + +function chunkFile(absPath, rel) { + const text = fs.readFileSync(absPath, 'utf8'); + const kind = classify(rel); + let raw = []; + + if (kind === 'header' && rel.endsWith('.sh')) { + raw = sectionsFromCommentHeader(text, '#', BASH_SECTIONS) + .map(s => ({ section: s.section, heading: s.title || null, content: s.content })); + } else if (kind === 'header' && rel.endsWith('.php')) { + raw = sectionsFromCommentHeader(text, '//', PHP_SECTIONS) + .map(s => ({ section: s.section, heading: s.title || null, content: s.content })); + } else if (kind === 'template') { + raw = sectionsFromConfTemplate(text) + .map(s => ({ section: null, heading: s.heading, content: s.content })); + } else if (kind === 'readme' || kind === 'manual' || kind === 'doc') { + raw = sectionsFromMarkdown(text) + .map(s => ({ section: null, heading: s.heading, content: s.content })); + } + + return capSize(raw).map(c => ({ + path: rel, + kind, + section: c.section || null, + heading: c.heading || null, + part: c.part || null, + content: c.content, + })); +} + +module.exports = { chunkFile, classify, BASH_SECTIONS, PHP_SECTIONS }; diff --git a/AI/lib/cli.js b/AI/lib/cli.js new file mode 100644 index 0000000..425a4b6 --- /dev/null +++ b/AI/lib/cli.js @@ -0,0 +1,176 @@ +'use strict'; +// ═══════════════════════════════════════════════════════════════════════════════════════════════ +// CLI bridge — the thin layer the bash entry points call. +// +// The bash scripts own configuration, gating, locking and logging, exactly as they do for every +// other Varaverk job. This file owns only the work that is genuinely awkward in bash: float +// vector math and SQLite BLOBs. That split follows the existing api_cache_writer.sh precedent — +// a bash shim in front of the language that fits the task. +// +// Every value arrives as an argument or an environment variable read by the caller. This file +// never reads a conf file itself, so there is exactly one place that decides what the settings +// are. +// ═══════════════════════════════════════════════════════════════════════════════════════════════ + +const { buildIndex } = require('./index.js'); +const { search } = require('./search.js'); + +function arg(name, dflt) { + const p = `--${name}=`; + const hit = process.argv.find(a => a.startsWith(p)); + return hit ? hit.slice(p.length) : dflt; +} +function flag(name) { + return process.argv.includes(`--${name}`); +} + +function fail(msg, code = 1) { + console.error(msg); + process.exit(code); +} + +async function cmdIndex() { + const root = arg('root'); + const db = arg('db'); + const url = arg('url'); + const model = arg('model', 'nomic-embed-text'); + if (!root || !db || !url) fail('index: --root, --db and --url are required'); + + const quiet = flag('quiet'); + let stats; + try { + stats = await buildIndex({ + root, dbPath: db, url, model, + batch: parseInt(arg('batch', '32'), 10), + timeout: parseInt(arg('timeout', '120000'), 10), + force: flag('force'), + dryRun: flag('dry-run'), + onProgress: p => { + if (p.error) console.error(`embed batch failed: ${p.error}`); + else if (!quiet && p.done % 320 === 0) console.log(` embedded ${p.done}/${p.total}`); + }, + }); + } catch (e) { + fail(`index failed: ${e.message}`, 2); + } + + if (flag('json')) { console.log(JSON.stringify(stats)); return; } + if (stats.dryRun) { + console.log(`DRY RUN — ${stats.files} file(s) would be indexed, ${stats.chunks} chunk(s) embedded`); + console.log(` ${stats.skipped} unchanged, ${stats.removed} stale entr(ies) would be dropped`); + return; + } + console.log(`indexed ${stats.files} file(s), ${stats.chunks} chunk(s) embedded`); + console.log(` ${stats.skipped} unchanged, ${stats.removed} removed, ${stats.failed} failed`); + console.log(` index now holds ${stats.total} chunk(s)`); + // A partial index is usable but not complete — say so in the exit code so a caller can act. + if (stats.failed) process.exit(3); +} + +async function cmdSearch() { + const db = arg('db'); + const url = arg('url'); + const model = arg('model', 'nomic-embed-text'); + const q = arg('query'); + if (!db || !url || !q) fail('search: --db, --url and --query are required'); + + let r; + try { + r = await search({ + dbPath: db, url, embedModel: model, query: q, + k: parseInt(arg('k', '8'), 10), + perFile: parseInt(arg('per-file', '3'), 10), + section: arg('section', null), + kind: arg('kind', null), + }); + } catch (e) { + fail(`search failed: ${e.message}`, 2); + } + + if (flag('json')) { console.log(JSON.stringify(r)); return; } + if (!r.results.length) { console.log('no matches'); return; } + if (r.intents.length) console.log(`intent: ${r.intents.join(', ')}\n`); + for (const x of r.results) { + const label = x.heading || x.section || '-'; + console.log(`── ${x.score.toFixed(3)} ${x.path} [${x.section || x.kind}] ${label}`); + console.log(x.content.split('\n').map(l => ' ' + l).join('\n')); + console.log(''); + } +} + +// Retrieval + generation. The prompt is built here so the context block and the instructions +// stay in one reviewable place. +async function cmdAsk() { + const db = arg('db'); + const url = arg('url'); + const embed = arg('embed-model', 'nomic-embed-text'); + const gen = arg('model'); + const q = arg('query'); + const timeout = parseInt(arg('timeout', '240000'), 10); + if (!db || !url || !gen || !q) fail('ask: --db, --url, --model and --query are required'); + + let r; + try { + r = await search({ + dbPath: db, url, embedModel: embed, query: q, + k: parseInt(arg('k', '6'), 10), perFile: parseInt(arg('per-file', '2'), 10), + }); + } catch (e) { + fail(`retrieval failed: ${e.message}`, 2); + } + if (!r.results.length) fail('no relevant context found in the index', 4); + + const context = r.results.map((x, i) => { + const label = [x.path, x.section, x.heading].filter(Boolean).join(' › '); + return `[${i + 1}] ${label}\n${x.content}`; + }).join('\n\n'); + + const prompt = +`You are answering questions about Varaverk, a two-server self-healing home media ecosystem. + +Answer ONLY from the context below. If the context does not contain the answer, say so plainly +and name what is missing — do not fill the gap from general knowledge about Linux, Docker or +rsync, because this system's conventions are frequently not the conventional ones. + +Cite the source of each claim as [n]. Be concise and concrete. + +CONTEXT +${context} + +QUESTION +${q} + +ANSWER`; + + let res; + try { + res = await fetch(`${url.replace(/\/$/, '')}/api/generate`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: gen, prompt, stream: false, + options: { temperature: 0.2, num_ctx: 8192 }, + }), + signal: AbortSignal.timeout(timeout), + }); + } catch (e) { + fail(`generation failed: ${e.message}`, 2); + } + if (!res.ok) fail(`generation HTTP ${res.status}`, 2); + const j = await res.json(); + + if (flag('json')) { + console.log(JSON.stringify({ answer: j.response, sources: r.results.map(x => ({ path: x.path, section: x.section, heading: x.heading, score: x.score })) })); + return; + } + console.log((j.response || '').trim()); + console.log('\nSources:'); + r.results.forEach((x, i) => { + console.log(` [${i + 1}] ${[x.path, x.section, x.heading].filter(Boolean).join(' › ')}`); + }); +} + +const cmd = process.argv[2]; +const table = { index: cmdIndex, search: cmdSearch, ask: cmdAsk }; +if (!table[cmd]) fail(`usage: cli.js [--flags]`); +table[cmd]().catch(e => fail(e.message, 2)); diff --git a/AI/lib/index.js b/AI/lib/index.js new file mode 100644 index 0000000..158203c --- /dev/null +++ b/AI/lib/index.js @@ -0,0 +1,199 @@ +'use strict'; +// ═══════════════════════════════════════════════════════════════════════════════════════════════ +// Indexer — chunk the repo, embed each chunk, store vectors in SQLite. +// +// Incremental by file mtime: a file whose mtime has not moved since its last index is skipped +// entirely, so a routine re-index costs seconds rather than re-embedding the whole corpus. +// +// Vectors are stored as raw little-endian float32 BLOBs. nomic-embed-text returns L2-normalised +// vectors, so cosine similarity is a plain dot product at query time — no normalising, no +// magnitude cache. PHP can read the same blobs with unpack('f*', $blob) when the UI needs them. +// ═══════════════════════════════════════════════════════════════════════════════════════════════ + +const fs = require('fs'); +const path = require('path'); +const { execSync } = require('child_process'); +const { DatabaseSync } = require('node:sqlite'); +const { chunkFile, classify } = require('./chunk.js'); + +const SCHEMA = ` +CREATE TABLE IF NOT EXISTS vv_files ( + path TEXT PRIMARY KEY, + mtime INTEGER NOT NULL, + chunks INTEGER NOT NULL, + indexed INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS vv_chunks ( + id INTEGER PRIMARY KEY, + path TEXT NOT NULL, + kind TEXT NOT NULL, + section TEXT, + heading TEXT, + part INTEGER, + content TEXT NOT NULL, + vector BLOB NOT NULL, + indexed INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_chunks_path ON vv_chunks(path); +CREATE INDEX IF NOT EXISTS idx_chunks_section ON vv_chunks(section); +CREATE INDEX IF NOT EXISTS idx_chunks_kind ON vv_chunks(kind); +CREATE TABLE IF NOT EXISTS vv_meta (k TEXT PRIMARY KEY, v TEXT); +`; + +function openDb(dbPath) { + fs.mkdirSync(path.dirname(dbPath), { recursive: true }); + const db = new DatabaseSync(dbPath); + db.exec('PRAGMA journal_mode = WAL;'); + db.exec('PRAGMA synchronous = NORMAL;'); + db.exec(SCHEMA); + return db; +} + +// Only ever index what git tracks. Configurations/, State_Files/ and data/ are gitignored, which +// is what makes it structurally impossible for a credential to reach the index — the files that +// hold them were never in the repo. Do not replace this with a filesystem walk. +function trackedFiles(root) { + return execSync('git ls-files', { cwd: root, maxBuffer: 1 << 26 }) + .toString().trim().split('\n') + .filter(Boolean) + .filter(f => classify(f) !== 'other'); +} + +async function embedBatch(url, model, inputs, timeoutMs) { + const ctl = AbortSignal.timeout(timeoutMs); + const res = await fetch(`${url.replace(/\/$/, '')}/api/embed`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ model, input: inputs }), + signal: ctl, + }); + if (!res.ok) throw new Error(`embed HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`); + const j = await res.json(); + if (!j.embeddings || j.embeddings.length !== inputs.length) + throw new Error(`embed returned ${j.embeddings ? j.embeddings.length : 0} of ${inputs.length}`); + return j.embeddings; +} + +function toBlob(vec) { + return Buffer.from(Float32Array.from(vec).buffer); +} + +async function buildIndex(opts) { + const { + root, dbPath, url, model, + batch = 32, timeout = 120000, force = false, dryRun = false, + onProgress = () => {}, + } = opts; + + const db = dryRun ? null : openDb(dbPath); + const now = Math.floor(Date.now() / 1000); + + const known = new Map(); + if (db) for (const r of db.prepare('SELECT path, mtime FROM vv_files').all()) known.set(r.path, r.mtime); + + const files = trackedFiles(root); + const present = new Set(files); + + // Files that left the repo must leave the index with them. + let removed = 0; + if (db && !force) { + for (const p of known.keys()) { + if (!present.has(p)) { + db.prepare('DELETE FROM vv_chunks WHERE path = ?').run(p); + db.prepare('DELETE FROM vv_files WHERE path = ?').run(p); + removed++; + } + } + } + if (db && force) { db.exec('DELETE FROM vv_chunks; DELETE FROM vv_files;'); } + + // ── Collect the chunks that actually need embedding ─────────────────────────────────────── + const pending = []; + let skipped = 0, scanned = 0; + + for (const rel of files) { + const abs = path.join(root, rel); + let st; + try { st = fs.statSync(abs); } catch { continue; } + const mtime = Math.floor(st.mtimeMs / 1000); + scanned++; + + if (!force && known.has(rel) && known.get(rel) === mtime) { skipped++; continue; } + + let chunks = []; + try { chunks = chunkFile(abs, rel); } catch (e) { continue; } + if (!chunks.length) { + if (db) { + db.prepare('DELETE FROM vv_chunks WHERE path = ?').run(rel); + db.prepare('INSERT OR REPLACE INTO vv_files VALUES (?,?,?,?)').run(rel, mtime, 0, now); + } + continue; + } + pending.push({ rel, mtime, chunks }); + } + + const totalChunks = pending.reduce((n, f) => n + f.chunks.length, 0); + if (dryRun) { + return { dryRun: true, scanned, skipped, removed, files: pending.length, chunks: totalChunks }; + } + + // ── Embed in batches, write per file so an interrupted run leaves a consistent index ─────── + const flat = []; + for (const f of pending) for (const c of f.chunks) flat.push({ f, c }); + + let done = 0, failed = 0; + for (let i = 0; i < flat.length; i += batch) { + const slice = flat.slice(i, i + batch); + const inputs = slice.map(x => x.c.content); + let vecs; + try { + vecs = await embedBatch(url, model, inputs, timeout); + } catch (e) { + failed += slice.length; + onProgress({ done, total: flat.length, error: e.message }); + continue; + } + slice.forEach((x, k) => { x.c.__vec = vecs[k]; }); + done += slice.length; + onProgress({ done, total: flat.length }); + } + + const ins = db.prepare( + 'INSERT INTO vv_chunks (path,kind,section,heading,part,content,vector,indexed) VALUES (?,?,?,?,?,?,?,?)' + ); + const insF = db.prepare('INSERT OR REPLACE INTO vv_files VALUES (?,?,?,?)'); + + db.exec('BEGIN'); + try { + for (const f of pending) { + const embedded = f.chunks.filter(c => c.__vec); + // A file whose chunks all failed to embed keeps its previous rows and its old mtime, + // so the next run retries it rather than recording a half-indexed file as current. + if (!embedded.length) continue; + db.prepare('DELETE FROM vv_chunks WHERE path = ?').run(f.rel); + for (const c of embedded) { + ins.run(c.path, c.kind, c.section, c.heading, c.part, c.content, toBlob(c.__vec), now); + } + insF.run(f.rel, f.mtime, embedded.length, now); + } + db.prepare('INSERT OR REPLACE INTO vv_meta VALUES (?,?)').run('embed_model', model); + db.prepare('INSERT OR REPLACE INTO vv_meta VALUES (?,?)').run('last_index', String(now)); + db.prepare('INSERT OR REPLACE INTO vv_meta VALUES (?,?)').run('dims', '768'); + db.exec('COMMIT'); + } catch (e) { + db.exec('ROLLBACK'); + throw e; + } + + const stats = { + scanned, skipped, removed, + files: pending.length, + chunks: done, + failed, + total: db.prepare('SELECT COUNT(*) n FROM vv_chunks').get().n, + }; + db.close(); + return stats; +} + +module.exports = { buildIndex, openDb, toBlob, trackedFiles }; diff --git a/AI/lib/search.js b/AI/lib/search.js new file mode 100644 index 0000000..8875dc5 --- /dev/null +++ b/AI/lib/search.js @@ -0,0 +1,121 @@ +'use strict'; +// ═══════════════════════════════════════════════════════════════════════════════════════════════ +// Search — embed a question, score it against the index, return the best chunks. +// +// nomic-embed-text returns L2-normalised vectors, so cosine similarity is a plain dot product. +// At this corpus size (~1.7k chunks x 768 dims) that is a couple of million multiply-adds — +// under a millisecond, with no vector database and no index structure to maintain. +// +// Section routing is the payoff from the header audit. Every chunk knows whether it is a +// PURPOSE, a DESIGN PRINCIPLES, an OPERATIONAL SAFEGUARDS and so on, so a question's shape can +// steer retrieval before similarity is even considered. It is applied as a score boost rather +// than a hard filter — intent detection is a heuristic, and a heuristic should not be able to +// exclude the one chunk that actually holds the answer. +// ═══════════════════════════════════════════════════════════════════════════════════════════════ + +const { DatabaseSync } = require('node:sqlite'); + +// Question shape → the section most likely to answer it. +const INTENTS = [ + { section: 'OPERATIONAL SAFEGUARDS', + re: /\b(safe|safety|guard|protect|prevent|fail|failure|abort|refuse|lock|root|timeout|dry.?run|what stops|what happens if|race|corrupt|data.?loss)\b/i }, + { section: 'CONFIGURATION', + re: /\b(variable|var|setting|conf|config|threshold|toggle|which key|what controls|where is .* set|default value|env)\b/i }, + { section: 'RUNTIME MODES', + re: /\b(flag|argument|option|--\w+|how do i run|invoke|cli|command line|status mode|usage)\b/i }, + { section: 'DESIGN PRINCIPLES', + re: /\b(why|rationale|reason|design|decision|deliberate|intentional|on purpose|trade.?off|chose|approach)\b/i }, + { section: 'OPERATIONAL MODEL', + re: /\b(how does .* work|flow|sequence|order|tier|lifecycle|state machine|when does)\b/i }, + { section: 'EXPORTS', + re: /\b(function|export|api surface|what does .* provide|helper|vv_\w+)\b/i }, + { section: 'PURPOSE', + re: /\b(what is|what does .* do|purpose|responsible for|job of)\b/i }, +]; + +const SECTION_BOOST = 0.06; // enough to reorder near-ties, not enough to beat a real match +const KIND_BOOST = 0.02; // docs answer "how do I" better than a script header does + +function detectIntent(q) { + const hits = []; + for (const i of INTENTS) if (i.re.test(q)) hits.push(i.section); + return hits; +} + +function blobToVec(buf) { + const b = Buffer.from(buf); + return new Float32Array(b.buffer, b.byteOffset, b.length / 4); +} + +function dot(a, b) { + let s = 0; + for (let i = 0; i < a.length; i++) s += a[i] * b[i]; + return s; +} + +async function embedQuery(url, model, text, timeoutMs = 60000) { + const res = await fetch(`${url.replace(/\/$/, '')}/api/embed`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ model, input: text }), + signal: AbortSignal.timeout(timeoutMs), + }); + if (!res.ok) throw new Error(`embed HTTP ${res.status}`); + const j = await res.json(); + if (!j.embeddings || !j.embeddings[0]) throw new Error('embed returned no vector'); + return Float32Array.from(j.embeddings[0]); +} + +// Keep at most `perFile` chunks from any one file, so a single large document cannot fill the +// entire context window and crowd out a better answer living somewhere else. +function diversify(rows, k, perFile) { + const seen = new Map(); + const out = []; + for (const r of rows) { + const n = seen.get(r.path) || 0; + if (n >= perFile) continue; + seen.set(r.path, n + 1); + out.push(r); + if (out.length >= k) break; + } + return out; +} + +async function search(opts) { + const { + dbPath, url, embedModel, query, + k = 8, perFile = 3, section = null, kind = null, minScore = 0.0, + } = opts; + + const db = new DatabaseSync(dbPath, { readOnly: true }); + + let sql = 'SELECT id,path,kind,section,heading,part,content,vector FROM vv_chunks'; + const where = [], args = []; + if (section) { where.push('section = ?'); args.push(section); } + if (kind) { where.push('kind = ?'); args.push(kind); } + if (where.length) sql += ' WHERE ' + where.join(' AND '); + + const rows = db.prepare(sql).all(...args); + if (!rows.length) { db.close(); return { results: [], intents: [], scanned: 0 }; } + + const qv = await embedQuery(url, embedModel, query); + const intents = section ? [] : detectIntent(query); + const wantDoc = /\b(how do i|steps|procedure|setup|install|troubleshoot|guide)\b/i.test(query); + + const scored = rows.map(r => { + let s = dot(qv, blobToVec(r.vector)); + if (intents.includes(r.section)) s += SECTION_BOOST; + if (wantDoc && (r.kind === 'manual' || r.kind === 'readme')) s += KIND_BOOST; + return { + id: r.id, path: r.path, kind: r.kind, section: r.section, + heading: r.heading, part: r.part, content: r.content, score: s, + }; + }); + + scored.sort((a, b) => b.score - a.score); + const kept = diversify(scored.filter(r => r.score >= minScore), k, perFile); + db.close(); + return { results: kept, intents, scanned: rows.length }; +} + +module.exports = { search, detectIntent, blobToVec, dot, embedQuery, INTENTS }; diff --git a/Fallback/fallback_test.sh b/Fallback/fallback_test.sh index 17299c2..30c070f 100755 --- a/Fallback/fallback_test.sh +++ b/Fallback/fallback_test.sh @@ -91,13 +91,13 @@ # FALLBACK_TEST_BLOCK_WAIT # Seconds to wait in Phase 3 for fallback.sh to detect the outage. # Must be > FALLBACK_CHECK_INTERVAL + buffer. At 30s interval: use ≥60s. -# (default: 60) +# (shipped default: 150) # # FALLBACK_TEST_HANDBACK_WAIT # Seconds to wait in Phase 6 for fallback.sh to complete handback. # Must cover: FALLBACK_HANDBACK_STRIKES × FALLBACK_CHECK_INTERVAL + rsync # duration + container start time. At 3 strikes × 30s + ~2min rsync + -# ~1min container start: use ≥240s. (default: 300) +# ~1min container start: use ≥240s. (shipped default: 360) # # ============================================================================================== # RUNTIME MODES diff --git a/Plugin/unraid/System_Essentials/mover_stop.sh b/Plugin/unraid/System_Essentials/mover_stop.sh index dcecbd0..ddbb53d 100755 --- a/Plugin/unraid/System_Essentials/mover_stop.sh +++ b/Plugin/unraid/System_Essentials/mover_stop.sh @@ -74,7 +74,7 @@ # master.conf # # MOVER_STOP_TIMEOUT -# Seconds between wall warning and SIGTERM. (default: 30) +# Seconds between wall warning and SIGTERM. (shipped default: 300) # # ============================================================================================== # RUNTIME MODES diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index 9412e06..daaa2da 100755 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -161,14 +161,14 @@ # Default retry attempts on rsync failure. (default: 3) # # SLEEP -# Default seconds between retry attempts. (default: 60) +# Default seconds between retry attempts. (shipped default: 300) # # RSYNC_MAX_RUNTIME_HOURS # Max hours a single transfer attempt may run before it's terminated and paused # for the next scheduled run. Protects the per-profile lock from being held # indefinitely by one huge/stuck transfer, starving other profiles of a turn. # Safe because DEFAULT_RSYNC_OPTS includes --partial — a paused transfer resumes -# from where it left off, not from scratch. (default: 23) +# from where it left off, not from scratch. (code fallback 23; shipped conf sets 19) # # ROOTFS_WARN # Abort threshold for remote rootfs percentage full. (default: 75) diff --git a/System_Essentials/server_reboot.sh b/System_Essentials/server_reboot.sh index fa4d578..3ca5434 100755 --- a/System_Essentials/server_reboot.sh +++ b/System_Essentials/server_reboot.sh @@ -85,7 +85,7 @@ # master.conf # # REBOOT_SLEEP -# Seconds between warning and shutdown sequence start. (default: 30) +# Seconds between warning and shutdown sequence start. (shipped default: 300) # # REBOOT_VM_WAIT # Seconds to wait for VMs to shut down gracefully. (default: 30)