From b00688bad1eaac4b174abe1de7258259b00b269a Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sun, 2 Aug 2026 01:13:01 -0400 Subject: [PATCH] Add RAG retrieval core, and correct six stale default values in headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chunker splits on the header sections the audit standardised, then sub-splits named-paragraph safeguards — without that a specific question about one of rsync.sh's fourteen safeguards scored below unrelated chunks, because the other thirteen dominated the vector. Index is SQLite with raw float32 blobs and is incremental on mtime; a no-op re-index takes 66ms. The stale defaults were found by asking the system a question and checking its answer: it correctly reported what mover_stop.sh's header claimed, and the header was wrong. --- AI/.gitignore | 4 + AI/lib/chunk.js | 255 ++++++++++++++++++ AI/lib/cli.js | 176 ++++++++++++ AI/lib/index.js | 199 ++++++++++++++ AI/lib/search.js | 121 +++++++++ Fallback/fallback_test.sh | 4 +- Plugin/unraid/System_Essentials/mover_stop.sh | 2 +- Rsync/rsync.sh | 4 +- System_Essentials/server_reboot.sh | 2 +- 9 files changed, 761 insertions(+), 6 deletions(-) create mode 100644 AI/.gitignore create mode 100644 AI/lib/chunk.js create mode 100644 AI/lib/cli.js create mode 100644 AI/lib/index.js create mode 100644 AI/lib/search.js diff --git a/AI/.gitignore b/AI/.gitignore new file mode 100644 index 0000000..936a447 --- /dev/null +++ b/AI/.gitignore @@ -0,0 +1,4 @@ +# The index is generated, host-specific, and regenerable in minutes. Never commit it. +*.db +*.db-wal +*.db-shm diff --git a/AI/lib/chunk.js b/AI/lib/chunk.js new file mode 100644 index 0000000..0de7aa1 --- /dev/null +++ b/AI/lib/chunk.js @@ -0,0 +1,255 @@ +'use strict'; +// ═══════════════════════════════════════════════════════════════════════════════════════════════ +// Chunker — turns repo files into retrieval units. +// +// The whole point of the header audit is that chunk boundaries are deterministic here. Bash +// scripts split on their six section names, markdown on its headings, conf templates on their +// ━━━ section rules. Nothing is split on a fixed token window, so no chunk ever contains half +// of one idea and half of another. +// +// Every chunk carries its section name as its own field, because that is the metadata that +// lets retrieval filter by question shape before it ever computes similarity. +// ═══════════════════════════════════════════════════════════════════════════════════════════════ + +const fs = require('fs'); +const path = require('path'); + +const BASH_SECTIONS = [ + 'PURPOSE', 'OPERATIONAL MODEL', 'DESIGN PRINCIPLES', + 'OPERATIONAL SAFEGUARDS', 'CONFIGURATION', 'RUNTIME MODES', +]; + +// PHP headers reuse the first three names deliberately, then diverge per layer. +const PHP_SECTIONS = [ + 'PURPOSE', 'OPERATIONAL MODEL', 'DESIGN PRINCIPLES', 'OPERATIONAL SAFEGUARDS', + 'STATUS', 'EXPORTS', 'REQUEST CONTRACT', 'SIDE EFFECTS', 'RENDERS', 'DEPENDS ON', + 'CONFIGURATION', +]; + +const MIN_CHARS = 40; // below this a chunk carries no retrievable meaning +const MAX_CHARS = 6000; // above this, split on blank lines — protects the embed window + +// Banner rules and box-drawing art are everywhere in this repo's headers. They carry no +// meaning to embed, and a chunk that is mostly rule characters is pure noise in the index. +// Measure a chunk by what is left after the decoration is removed, not by raw length. +function meaningful(s) { + return s.replace(/[═─━=_#\/*\s|+.-]/g, '').length; +} +const MIN_MEANINGFUL = 30; + +function isRealHeading(h) { + return !!h && /[A-Za-z0-9]/.test(h.replace(/[═─━=_]/g, '')); +} + +function stripPrefix(line, prefix) { + // '# text' -> 'text' '// text' -> 'text' + const re = new RegExp('^\\s*' + prefix + '\\s?'); + return line.replace(re, ''); +} + +// ── Comment-header sectioning, shared by bash (#) and PHP (//) ──────────────────────────────── +function sectionsFromCommentHeader(text, prefix, names) { + const lines = text.split('\n'); + const nameSet = new Set(names); + const found = []; + + // headerEnd matters as much as the section starts. The last section (RUNTIME MODES in bash, + // DEPENDS ON in a page) would otherwise run to EOF and sweep up every unrelated comment in + // the file — scheduler.php alone contributed an 11k-char chunk of unrelated inline comments. + let headerEnd = lines.length; + for (let i = 0; i < lines.length; i++) { + const raw = lines[i]; + if (!new RegExp('^\\s*' + prefix).test(raw)) { + // Header block ends at the first non-comment, non-blank line past the shebang. + // '' bracket a PHP header block and are not the end of it. + const t = raw.trim(); + if (t !== '' && !/^#!/.test(t) && t !== '' && found.length) { + headerEnd = i; + break; + } + continue; + } + const inner = stripPrefix(raw, prefix).trim(); + if (nameSet.has(inner)) found.push({ name: inner, start: i }); + } + + const out = []; + for (let k = 0; k < found.length; k++) { + const start = found[k].start + 1; + const end = k + 1 < found.length ? found[k + 1].start : headerEnd; + const body = lines.slice(start, end) + .filter(l => new RegExp('^\\s*' + prefix).test(l)) + .map(l => stripPrefix(l, prefix)) + // drop pure separator rules (════, ────, ━━━) — they carry no meaning + .filter(l => !/^[\s═─━=_-]*$/.test(l) || l.trim() === '') + .join('\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); + if (meaningful(body) >= MIN_MEANINGFUL) { + for (const p of splitNamedParagraphs(body)) + out.push({ section: found[k].name, title: p.title, content: p.content }); + } + } + return out; +} + +// ── Named-paragraph sub-chunking ────────────────────────────────────────────────────────────── +// The header convention writes safeguards and principles as named paragraphs: an unindented +// title line followed by an indented body. Embedding a whole section as one unit dilutes them — +// rsync.sh's OPERATIONAL SAFEGUARDS holds eight distinct guarantees in 2.8k chars, and a query +// about one of them scored below unrelated chunks because the other seven dominated the vector. +// Splitting on the title lines is what makes a specific question find a specific answer. +// +// The section name is carried onto every sub-chunk, so section routing still works; the +// paragraph title becomes the chunk's heading. +function splitNamedParagraphs(body) { + const lines = body.split('\n'); + const marks = []; + for (let i = 0; i < lines.length; i++) { + const l = lines[i]; + if (!l.trim()) continue; + if (/^\s/.test(l)) continue; // indented => body, not a title + if (/^[-*•]/.test(l.trim())) continue; // list item, not a title + if (l.trim().length > 80) continue; // a long line is prose, not a heading + if (/[.:;,]$/.test(l.trim())) continue; // ends like a sentence + + // The decisive signal: a real title is followed by an indented body. Wrapped prose is + // followed by more unindented prose. Without this check, any short line in a paragraph + // that happened to wrap became a spurious chunk boundary mid-sentence. + let j = i + 1; + while (j < lines.length && !lines[j].trim()) j++; + if (j >= lines.length || !/^\s+\S/.test(lines[j])) continue; + + marks.push(i); + } + // Fewer than two titles means this section is not written as named paragraphs — keep it whole. + if (marks.length < 2) return [{ title: null, content: body }]; + + const out = []; + if (marks[0] > 0) { + const pre = lines.slice(0, marks[0]).join('\n').trim(); + if (meaningful(pre) >= MIN_MEANINGFUL) out.push({ title: null, content: pre }); + } + for (let k = 0; k < marks.length; k++) { + const start = marks[k]; + const end = k + 1 < marks.length ? marks[k + 1] : lines.length; + const title = lines[start].trim(); + const content = lines.slice(start, end).join('\n').trim(); + if (meaningful(content) >= MIN_MEANINGFUL) out.push({ title, content }); + } + return out; +} + +// ── Markdown: split on ## headings, keep the heading with its body ───────────────────────────── +function sectionsFromMarkdown(text) { + const lines = text.split('\n'); + const marks = []; + let fence = false; + + for (let i = 0; i < lines.length; i++) { + if (/^\s*```/.test(lines[i])) { fence = !fence; continue; } + if (fence) continue; + if (/^#{1,3}\s+\S/.test(lines[i])) marks.push(i); + } + if (!marks.length) return [{ heading: null, content: text.trim() }]; + + const out = []; + // preamble before the first heading + if (marks[0] > 0) { + const pre = lines.slice(0, marks[0]).join('\n').trim(); + if (meaningful(pre) >= MIN_MEANINGFUL) out.push({ heading: null, content: pre }); + } + for (let k = 0; k < marks.length; k++) { + const start = marks[k]; + const end = k + 1 < marks.length ? marks[k + 1] : lines.length; + const heading = lines[start].replace(/^#+\s*/, '').replace(/[━─═]+/g, '').trim(); + const content = lines.slice(start, end).join('\n').trim(); + if (meaningful(content) >= MIN_MEANINGFUL) out.push({ heading: isRealHeading(heading) ? heading : null, content }); + } + return out; +} + +// ── Conf templates: split on the ━━━ / ── section rules ─────────────────────────────────────── +function sectionsFromConfTemplate(text) { + const lines = text.split('\n'); + const marks = []; + for (let i = 0; i < lines.length; i++) { + const m = lines[i].match(/^#\s*[━─]{2,}\s*(.+?)\s*[━─]{2,}\s*$/); + if (m && isRealHeading(m[1])) marks.push({ i, name: m[1].trim() }); + } + if (!marks.length) return []; + + const out = []; + for (let k = 0; k < marks.length; k++) { + const start = marks[k].i; + const end = k + 1 < marks.length ? marks[k + 1].i : lines.length; + const content = lines.slice(start, end).join('\n').replace(/\n{3,}/g, '\n\n').trim(); + if (meaningful(content) >= MIN_MEANINGFUL) out.push({ heading: marks[k].name, content }); + } + return out; +} + +// Oversized chunks split on blank lines rather than mid-sentence. +function capSize(chunks) { + const out = []; + for (const c of chunks) { + if (c.content.length <= MAX_CHARS) { out.push(c); continue; } + const paras = c.content.split(/\n\s*\n/); + let buf = [], len = 0, part = 1; + const flush = () => { + if (!buf.length) return; + out.push({ ...c, content: buf.join('\n\n'), part: part++ }); + buf = []; len = 0; + }; + for (const p of paras) { + if (len + p.length > MAX_CHARS && buf.length) flush(); + buf.push(p); len += p.length + 2; + } + flush(); + } + return out; +} + +function classify(rel) { + const base = path.basename(rel); + if (rel.startsWith('Deployment/') && rel.endsWith('.template')) return 'template'; + if (base.endsWith('.md')) { + if (base.startsWith('Manual')) return 'manual'; + if (base.startsWith('README') || base === 'README.md') return 'readme'; + return 'doc'; + } + if (base.endsWith('.sh')) return 'header'; + if (base.endsWith('.php')) return 'header'; + return 'other'; +} + +function chunkFile(absPath, rel) { + const text = fs.readFileSync(absPath, 'utf8'); + const kind = classify(rel); + let raw = []; + + if (kind === 'header' && rel.endsWith('.sh')) { + raw = sectionsFromCommentHeader(text, '#', BASH_SECTIONS) + .map(s => ({ section: s.section, heading: s.title || null, content: s.content })); + } else if (kind === 'header' && rel.endsWith('.php')) { + raw = sectionsFromCommentHeader(text, '//', PHP_SECTIONS) + .map(s => ({ section: s.section, heading: s.title || null, content: s.content })); + } else if (kind === 'template') { + raw = sectionsFromConfTemplate(text) + .map(s => ({ section: null, heading: s.heading, content: s.content })); + } else if (kind === 'readme' || kind === 'manual' || kind === 'doc') { + raw = sectionsFromMarkdown(text) + .map(s => ({ section: null, heading: s.heading, content: s.content })); + } + + return capSize(raw).map(c => ({ + path: rel, + kind, + section: c.section || null, + heading: c.heading || null, + part: c.part || null, + content: c.content, + })); +} + +module.exports = { chunkFile, classify, BASH_SECTIONS, PHP_SECTIONS }; diff --git a/AI/lib/cli.js b/AI/lib/cli.js new file mode 100644 index 0000000..425a4b6 --- /dev/null +++ b/AI/lib/cli.js @@ -0,0 +1,176 @@ +'use strict'; +// ═══════════════════════════════════════════════════════════════════════════════════════════════ +// CLI bridge — the thin layer the bash entry points call. +// +// The bash scripts own configuration, gating, locking and logging, exactly as they do for every +// other Varaverk job. This file owns only the work that is genuinely awkward in bash: float +// vector math and SQLite BLOBs. That split follows the existing api_cache_writer.sh precedent — +// a bash shim in front of the language that fits the task. +// +// Every value arrives as an argument or an environment variable read by the caller. This file +// never reads a conf file itself, so there is exactly one place that decides what the settings +// are. +// ═══════════════════════════════════════════════════════════════════════════════════════════════ + +const { buildIndex } = require('./index.js'); +const { search } = require('./search.js'); + +function arg(name, dflt) { + const p = `--${name}=`; + const hit = process.argv.find(a => a.startsWith(p)); + return hit ? hit.slice(p.length) : dflt; +} +function flag(name) { + return process.argv.includes(`--${name}`); +} + +function fail(msg, code = 1) { + console.error(msg); + process.exit(code); +} + +async function cmdIndex() { + const root = arg('root'); + const db = arg('db'); + const url = arg('url'); + const model = arg('model', 'nomic-embed-text'); + if (!root || !db || !url) fail('index: --root, --db and --url are required'); + + const quiet = flag('quiet'); + let stats; + try { + stats = await buildIndex({ + root, dbPath: db, url, model, + batch: parseInt(arg('batch', '32'), 10), + timeout: parseInt(arg('timeout', '120000'), 10), + force: flag('force'), + dryRun: flag('dry-run'), + onProgress: p => { + if (p.error) console.error(`embed batch failed: ${p.error}`); + else if (!quiet && p.done % 320 === 0) console.log(` embedded ${p.done}/${p.total}`); + }, + }); + } catch (e) { + fail(`index failed: ${e.message}`, 2); + } + + if (flag('json')) { console.log(JSON.stringify(stats)); return; } + if (stats.dryRun) { + console.log(`DRY RUN — ${stats.files} file(s) would be indexed, ${stats.chunks} chunk(s) embedded`); + console.log(` ${stats.skipped} unchanged, ${stats.removed} stale entr(ies) would be dropped`); + return; + } + console.log(`indexed ${stats.files} file(s), ${stats.chunks} chunk(s) embedded`); + console.log(` ${stats.skipped} unchanged, ${stats.removed} removed, ${stats.failed} failed`); + console.log(` index now holds ${stats.total} chunk(s)`); + // A partial index is usable but not complete — say so in the exit code so a caller can act. + if (stats.failed) process.exit(3); +} + +async function cmdSearch() { + const db = arg('db'); + const url = arg('url'); + const model = arg('model', 'nomic-embed-text'); + const q = arg('query'); + if (!db || !url || !q) fail('search: --db, --url and --query are required'); + + let r; + try { + r = await search({ + dbPath: db, url, embedModel: model, query: q, + k: parseInt(arg('k', '8'), 10), + perFile: parseInt(arg('per-file', '3'), 10), + section: arg('section', null), + kind: arg('kind', null), + }); + } catch (e) { + fail(`search failed: ${e.message}`, 2); + } + + if (flag('json')) { console.log(JSON.stringify(r)); return; } + if (!r.results.length) { console.log('no matches'); return; } + if (r.intents.length) console.log(`intent: ${r.intents.join(', ')}\n`); + for (const x of r.results) { + const label = x.heading || x.section || '-'; + console.log(`── ${x.score.toFixed(3)} ${x.path} [${x.section || x.kind}] ${label}`); + console.log(x.content.split('\n').map(l => ' ' + l).join('\n')); + console.log(''); + } +} + +// Retrieval + generation. The prompt is built here so the context block and the instructions +// stay in one reviewable place. +async function cmdAsk() { + const db = arg('db'); + const url = arg('url'); + const embed = arg('embed-model', 'nomic-embed-text'); + const gen = arg('model'); + const q = arg('query'); + const timeout = parseInt(arg('timeout', '240000'), 10); + if (!db || !url || !gen || !q) fail('ask: --db, --url, --model and --query are required'); + + let r; + try { + r = await search({ + dbPath: db, url, embedModel: embed, query: q, + k: parseInt(arg('k', '6'), 10), perFile: parseInt(arg('per-file', '2'), 10), + }); + } catch (e) { + fail(`retrieval failed: ${e.message}`, 2); + } + if (!r.results.length) fail('no relevant context found in the index', 4); + + const context = r.results.map((x, i) => { + const label = [x.path, x.section, x.heading].filter(Boolean).join(' › '); + return `[${i + 1}] ${label}\n${x.content}`; + }).join('\n\n'); + + const prompt = +`You are answering questions about Varaverk, a two-server self-healing home media ecosystem. + +Answer ONLY from the context below. If the context does not contain the answer, say so plainly +and name what is missing — do not fill the gap from general knowledge about Linux, Docker or +rsync, because this system's conventions are frequently not the conventional ones. + +Cite the source of each claim as [n]. Be concise and concrete. + +CONTEXT +${context} + +QUESTION +${q} + +ANSWER`; + + let res; + try { + res = await fetch(`${url.replace(/\/$/, '')}/api/generate`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: gen, prompt, stream: false, + options: { temperature: 0.2, num_ctx: 8192 }, + }), + signal: AbortSignal.timeout(timeout), + }); + } catch (e) { + fail(`generation failed: ${e.message}`, 2); + } + if (!res.ok) fail(`generation HTTP ${res.status}`, 2); + const j = await res.json(); + + if (flag('json')) { + console.log(JSON.stringify({ answer: j.response, sources: r.results.map(x => ({ path: x.path, section: x.section, heading: x.heading, score: x.score })) })); + return; + } + console.log((j.response || '').trim()); + console.log('\nSources:'); + r.results.forEach((x, i) => { + console.log(` [${i + 1}] ${[x.path, x.section, x.heading].filter(Boolean).join(' › ')}`); + }); +} + +const cmd = process.argv[2]; +const table = { index: cmdIndex, search: cmdSearch, ask: cmdAsk }; +if (!table[cmd]) fail(`usage: cli.js [--flags]`); +table[cmd]().catch(e => fail(e.message, 2)); diff --git a/AI/lib/index.js b/AI/lib/index.js new file mode 100644 index 0000000..158203c --- /dev/null +++ b/AI/lib/index.js @@ -0,0 +1,199 @@ +'use strict'; +// ═══════════════════════════════════════════════════════════════════════════════════════════════ +// Indexer — chunk the repo, embed each chunk, store vectors in SQLite. +// +// Incremental by file mtime: a file whose mtime has not moved since its last index is skipped +// entirely, so a routine re-index costs seconds rather than re-embedding the whole corpus. +// +// Vectors are stored as raw little-endian float32 BLOBs. nomic-embed-text returns L2-normalised +// vectors, so cosine similarity is a plain dot product at query time — no normalising, no +// magnitude cache. PHP can read the same blobs with unpack('f*', $blob) when the UI needs them. +// ═══════════════════════════════════════════════════════════════════════════════════════════════ + +const fs = require('fs'); +const path = require('path'); +const { execSync } = require('child_process'); +const { DatabaseSync } = require('node:sqlite'); +const { chunkFile, classify } = require('./chunk.js'); + +const SCHEMA = ` +CREATE TABLE IF NOT EXISTS vv_files ( + path TEXT PRIMARY KEY, + mtime INTEGER NOT NULL, + chunks INTEGER NOT NULL, + indexed INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS vv_chunks ( + id INTEGER PRIMARY KEY, + path TEXT NOT NULL, + kind TEXT NOT NULL, + section TEXT, + heading TEXT, + part INTEGER, + content TEXT NOT NULL, + vector BLOB NOT NULL, + indexed INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_chunks_path ON vv_chunks(path); +CREATE INDEX IF NOT EXISTS idx_chunks_section ON vv_chunks(section); +CREATE INDEX IF NOT EXISTS idx_chunks_kind ON vv_chunks(kind); +CREATE TABLE IF NOT EXISTS vv_meta (k TEXT PRIMARY KEY, v TEXT); +`; + +function openDb(dbPath) { + fs.mkdirSync(path.dirname(dbPath), { recursive: true }); + const db = new DatabaseSync(dbPath); + db.exec('PRAGMA journal_mode = WAL;'); + db.exec('PRAGMA synchronous = NORMAL;'); + db.exec(SCHEMA); + return db; +} + +// Only ever index what git tracks. Configurations/, State_Files/ and data/ are gitignored, which +// is what makes it structurally impossible for a credential to reach the index — the files that +// hold them were never in the repo. Do not replace this with a filesystem walk. +function trackedFiles(root) { + return execSync('git ls-files', { cwd: root, maxBuffer: 1 << 26 }) + .toString().trim().split('\n') + .filter(Boolean) + .filter(f => classify(f) !== 'other'); +} + +async function embedBatch(url, model, inputs, timeoutMs) { + const ctl = AbortSignal.timeout(timeoutMs); + const res = await fetch(`${url.replace(/\/$/, '')}/api/embed`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ model, input: inputs }), + signal: ctl, + }); + if (!res.ok) throw new Error(`embed HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`); + const j = await res.json(); + if (!j.embeddings || j.embeddings.length !== inputs.length) + throw new Error(`embed returned ${j.embeddings ? j.embeddings.length : 0} of ${inputs.length}`); + return j.embeddings; +} + +function toBlob(vec) { + return Buffer.from(Float32Array.from(vec).buffer); +} + +async function buildIndex(opts) { + const { + root, dbPath, url, model, + batch = 32, timeout = 120000, force = false, dryRun = false, + onProgress = () => {}, + } = opts; + + const db = dryRun ? null : openDb(dbPath); + const now = Math.floor(Date.now() / 1000); + + const known = new Map(); + if (db) for (const r of db.prepare('SELECT path, mtime FROM vv_files').all()) known.set(r.path, r.mtime); + + const files = trackedFiles(root); + const present = new Set(files); + + // Files that left the repo must leave the index with them. + let removed = 0; + if (db && !force) { + for (const p of known.keys()) { + if (!present.has(p)) { + db.prepare('DELETE FROM vv_chunks WHERE path = ?').run(p); + db.prepare('DELETE FROM vv_files WHERE path = ?').run(p); + removed++; + } + } + } + if (db && force) { db.exec('DELETE FROM vv_chunks; DELETE FROM vv_files;'); } + + // ── Collect the chunks that actually need embedding ─────────────────────────────────────── + const pending = []; + let skipped = 0, scanned = 0; + + for (const rel of files) { + const abs = path.join(root, rel); + let st; + try { st = fs.statSync(abs); } catch { continue; } + const mtime = Math.floor(st.mtimeMs / 1000); + scanned++; + + if (!force && known.has(rel) && known.get(rel) === mtime) { skipped++; continue; } + + let chunks = []; + try { chunks = chunkFile(abs, rel); } catch (e) { continue; } + if (!chunks.length) { + if (db) { + db.prepare('DELETE FROM vv_chunks WHERE path = ?').run(rel); + db.prepare('INSERT OR REPLACE INTO vv_files VALUES (?,?,?,?)').run(rel, mtime, 0, now); + } + continue; + } + pending.push({ rel, mtime, chunks }); + } + + const totalChunks = pending.reduce((n, f) => n + f.chunks.length, 0); + if (dryRun) { + return { dryRun: true, scanned, skipped, removed, files: pending.length, chunks: totalChunks }; + } + + // ── Embed in batches, write per file so an interrupted run leaves a consistent index ─────── + const flat = []; + for (const f of pending) for (const c of f.chunks) flat.push({ f, c }); + + let done = 0, failed = 0; + for (let i = 0; i < flat.length; i += batch) { + const slice = flat.slice(i, i + batch); + const inputs = slice.map(x => x.c.content); + let vecs; + try { + vecs = await embedBatch(url, model, inputs, timeout); + } catch (e) { + failed += slice.length; + onProgress({ done, total: flat.length, error: e.message }); + continue; + } + slice.forEach((x, k) => { x.c.__vec = vecs[k]; }); + done += slice.length; + onProgress({ done, total: flat.length }); + } + + const ins = db.prepare( + 'INSERT INTO vv_chunks (path,kind,section,heading,part,content,vector,indexed) VALUES (?,?,?,?,?,?,?,?)' + ); + const insF = db.prepare('INSERT OR REPLACE INTO vv_files VALUES (?,?,?,?)'); + + db.exec('BEGIN'); + try { + for (const f of pending) { + const embedded = f.chunks.filter(c => c.__vec); + // A file whose chunks all failed to embed keeps its previous rows and its old mtime, + // so the next run retries it rather than recording a half-indexed file as current. + if (!embedded.length) continue; + db.prepare('DELETE FROM vv_chunks WHERE path = ?').run(f.rel); + for (const c of embedded) { + ins.run(c.path, c.kind, c.section, c.heading, c.part, c.content, toBlob(c.__vec), now); + } + insF.run(f.rel, f.mtime, embedded.length, now); + } + db.prepare('INSERT OR REPLACE INTO vv_meta VALUES (?,?)').run('embed_model', model); + db.prepare('INSERT OR REPLACE INTO vv_meta VALUES (?,?)').run('last_index', String(now)); + db.prepare('INSERT OR REPLACE INTO vv_meta VALUES (?,?)').run('dims', '768'); + db.exec('COMMIT'); + } catch (e) { + db.exec('ROLLBACK'); + throw e; + } + + const stats = { + scanned, skipped, removed, + files: pending.length, + chunks: done, + failed, + total: db.prepare('SELECT COUNT(*) n FROM vv_chunks').get().n, + }; + db.close(); + return stats; +} + +module.exports = { buildIndex, openDb, toBlob, trackedFiles }; diff --git a/AI/lib/search.js b/AI/lib/search.js new file mode 100644 index 0000000..8875dc5 --- /dev/null +++ b/AI/lib/search.js @@ -0,0 +1,121 @@ +'use strict'; +// ═══════════════════════════════════════════════════════════════════════════════════════════════ +// Search — embed a question, score it against the index, return the best chunks. +// +// nomic-embed-text returns L2-normalised vectors, so cosine similarity is a plain dot product. +// At this corpus size (~1.7k chunks x 768 dims) that is a couple of million multiply-adds — +// under a millisecond, with no vector database and no index structure to maintain. +// +// Section routing is the payoff from the header audit. Every chunk knows whether it is a +// PURPOSE, a DESIGN PRINCIPLES, an OPERATIONAL SAFEGUARDS and so on, so a question's shape can +// steer retrieval before similarity is even considered. It is applied as a score boost rather +// than a hard filter — intent detection is a heuristic, and a heuristic should not be able to +// exclude the one chunk that actually holds the answer. +// ═══════════════════════════════════════════════════════════════════════════════════════════════ + +const { DatabaseSync } = require('node:sqlite'); + +// Question shape → the section most likely to answer it. +const INTENTS = [ + { section: 'OPERATIONAL SAFEGUARDS', + re: /\b(safe|safety|guard|protect|prevent|fail|failure|abort|refuse|lock|root|timeout|dry.?run|what stops|what happens if|race|corrupt|data.?loss)\b/i }, + { section: 'CONFIGURATION', + re: /\b(variable|var|setting|conf|config|threshold|toggle|which key|what controls|where is .* set|default value|env)\b/i }, + { section: 'RUNTIME MODES', + re: /\b(flag|argument|option|--\w+|how do i run|invoke|cli|command line|status mode|usage)\b/i }, + { section: 'DESIGN PRINCIPLES', + re: /\b(why|rationale|reason|design|decision|deliberate|intentional|on purpose|trade.?off|chose|approach)\b/i }, + { section: 'OPERATIONAL MODEL', + re: /\b(how does .* work|flow|sequence|order|tier|lifecycle|state machine|when does)\b/i }, + { section: 'EXPORTS', + re: /\b(function|export|api surface|what does .* provide|helper|vv_\w+)\b/i }, + { section: 'PURPOSE', + re: /\b(what is|what does .* do|purpose|responsible for|job of)\b/i }, +]; + +const SECTION_BOOST = 0.06; // enough to reorder near-ties, not enough to beat a real match +const KIND_BOOST = 0.02; // docs answer "how do I" better than a script header does + +function detectIntent(q) { + const hits = []; + for (const i of INTENTS) if (i.re.test(q)) hits.push(i.section); + return hits; +} + +function blobToVec(buf) { + const b = Buffer.from(buf); + return new Float32Array(b.buffer, b.byteOffset, b.length / 4); +} + +function dot(a, b) { + let s = 0; + for (let i = 0; i < a.length; i++) s += a[i] * b[i]; + return s; +} + +async function embedQuery(url, model, text, timeoutMs = 60000) { + const res = await fetch(`${url.replace(/\/$/, '')}/api/embed`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ model, input: text }), + signal: AbortSignal.timeout(timeoutMs), + }); + if (!res.ok) throw new Error(`embed HTTP ${res.status}`); + const j = await res.json(); + if (!j.embeddings || !j.embeddings[0]) throw new Error('embed returned no vector'); + return Float32Array.from(j.embeddings[0]); +} + +// Keep at most `perFile` chunks from any one file, so a single large document cannot fill the +// entire context window and crowd out a better answer living somewhere else. +function diversify(rows, k, perFile) { + const seen = new Map(); + const out = []; + for (const r of rows) { + const n = seen.get(r.path) || 0; + if (n >= perFile) continue; + seen.set(r.path, n + 1); + out.push(r); + if (out.length >= k) break; + } + return out; +} + +async function search(opts) { + const { + dbPath, url, embedModel, query, + k = 8, perFile = 3, section = null, kind = null, minScore = 0.0, + } = opts; + + const db = new DatabaseSync(dbPath, { readOnly: true }); + + let sql = 'SELECT id,path,kind,section,heading,part,content,vector FROM vv_chunks'; + const where = [], args = []; + if (section) { where.push('section = ?'); args.push(section); } + if (kind) { where.push('kind = ?'); args.push(kind); } + if (where.length) sql += ' WHERE ' + where.join(' AND '); + + const rows = db.prepare(sql).all(...args); + if (!rows.length) { db.close(); return { results: [], intents: [], scanned: 0 }; } + + const qv = await embedQuery(url, embedModel, query); + const intents = section ? [] : detectIntent(query); + const wantDoc = /\b(how do i|steps|procedure|setup|install|troubleshoot|guide)\b/i.test(query); + + const scored = rows.map(r => { + let s = dot(qv, blobToVec(r.vector)); + if (intents.includes(r.section)) s += SECTION_BOOST; + if (wantDoc && (r.kind === 'manual' || r.kind === 'readme')) s += KIND_BOOST; + return { + id: r.id, path: r.path, kind: r.kind, section: r.section, + heading: r.heading, part: r.part, content: r.content, score: s, + }; + }); + + scored.sort((a, b) => b.score - a.score); + const kept = diversify(scored.filter(r => r.score >= minScore), k, perFile); + db.close(); + return { results: kept, intents, scanned: rows.length }; +} + +module.exports = { search, detectIntent, blobToVec, dot, embedQuery, INTENTS }; diff --git a/Fallback/fallback_test.sh b/Fallback/fallback_test.sh index 17299c2..30c070f 100755 --- a/Fallback/fallback_test.sh +++ b/Fallback/fallback_test.sh @@ -91,13 +91,13 @@ # FALLBACK_TEST_BLOCK_WAIT # Seconds to wait in Phase 3 for fallback.sh to detect the outage. # Must be > FALLBACK_CHECK_INTERVAL + buffer. At 30s interval: use ≥60s. -# (default: 60) +# (shipped default: 150) # # FALLBACK_TEST_HANDBACK_WAIT # Seconds to wait in Phase 6 for fallback.sh to complete handback. # Must cover: FALLBACK_HANDBACK_STRIKES × FALLBACK_CHECK_INTERVAL + rsync # duration + container start time. At 3 strikes × 30s + ~2min rsync + -# ~1min container start: use ≥240s. (default: 300) +# ~1min container start: use ≥240s. (shipped default: 360) # # ============================================================================================== # RUNTIME MODES diff --git a/Plugin/unraid/System_Essentials/mover_stop.sh b/Plugin/unraid/System_Essentials/mover_stop.sh index dcecbd0..ddbb53d 100755 --- a/Plugin/unraid/System_Essentials/mover_stop.sh +++ b/Plugin/unraid/System_Essentials/mover_stop.sh @@ -74,7 +74,7 @@ # master.conf # # MOVER_STOP_TIMEOUT -# Seconds between wall warning and SIGTERM. (default: 30) +# Seconds between wall warning and SIGTERM. (shipped default: 300) # # ============================================================================================== # RUNTIME MODES diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index 9412e06..daaa2da 100755 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -161,14 +161,14 @@ # Default retry attempts on rsync failure. (default: 3) # # SLEEP -# Default seconds between retry attempts. (default: 60) +# Default seconds between retry attempts. (shipped default: 300) # # RSYNC_MAX_RUNTIME_HOURS # Max hours a single transfer attempt may run before it's terminated and paused # for the next scheduled run. Protects the per-profile lock from being held # indefinitely by one huge/stuck transfer, starving other profiles of a turn. # Safe because DEFAULT_RSYNC_OPTS includes --partial — a paused transfer resumes -# from where it left off, not from scratch. (default: 23) +# from where it left off, not from scratch. (code fallback 23; shipped conf sets 19) # # ROOTFS_WARN # Abort threshold for remote rootfs percentage full. (default: 75) diff --git a/System_Essentials/server_reboot.sh b/System_Essentials/server_reboot.sh index fa4d578..3ca5434 100755 --- a/System_Essentials/server_reboot.sh +++ b/System_Essentials/server_reboot.sh @@ -85,7 +85,7 @@ # master.conf # # REBOOT_SLEEP -# Seconds between warning and shutdown sequence start. (default: 30) +# Seconds between warning and shutdown sequence start. (shipped default: 300) # # REBOOT_VM_WAIT # Seconds to wait for VMs to shut down gracefully. (default: 30)