Gmer4Lfe
b00688bad1
Chunker splits on the header sections the audit standardised, then sub-splits named-paragraph safeguards — without that a specific question about one of rsync.sh's fourteen safeguards scored below unrelated chunks, because the other thirteen dominated the vector. Index is SQLite with raw float32 blobs and is incremental on mtime; a no-op re-index takes 66ms. The stale defaults were found by asking the system a question and checking its answer: it correctly reported what mover_stop.sh's header claimed, and the header was wrong.
256 lines
12 KiB
JavaScript
256 lines
12 KiB
JavaScript
'use strict';
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
// Chunker — turns repo files into retrieval units.
|
|
//
|
|
// The whole point of the header audit is that chunk boundaries are deterministic here. Bash
|
|
// scripts split on their six section names, markdown on its headings, conf templates on their
|
|
// ━━━ section rules. Nothing is split on a fixed token window, so no chunk ever contains half
|
|
// of one idea and half of another.
|
|
//
|
|
// Every chunk carries its section name as its own field, because that is the metadata that
|
|
// lets retrieval filter by question shape before it ever computes similarity.
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
const BASH_SECTIONS = [
|
|
'PURPOSE', 'OPERATIONAL MODEL', 'DESIGN PRINCIPLES',
|
|
'OPERATIONAL SAFEGUARDS', 'CONFIGURATION', 'RUNTIME MODES',
|
|
];
|
|
|
|
// PHP headers reuse the first three names deliberately, then diverge per layer.
|
|
const PHP_SECTIONS = [
|
|
'PURPOSE', 'OPERATIONAL MODEL', 'DESIGN PRINCIPLES', 'OPERATIONAL SAFEGUARDS',
|
|
'STATUS', 'EXPORTS', 'REQUEST CONTRACT', 'SIDE EFFECTS', 'RENDERS', 'DEPENDS ON',
|
|
'CONFIGURATION',
|
|
];
|
|
|
|
const MIN_CHARS = 40; // below this a chunk carries no retrievable meaning
|
|
const MAX_CHARS = 6000; // above this, split on blank lines — protects the embed window
|
|
|
|
// Banner rules and box-drawing art are everywhere in this repo's headers. They carry no
|
|
// meaning to embed, and a chunk that is mostly rule characters is pure noise in the index.
|
|
// Measure a chunk by what is left after the decoration is removed, not by raw length.
|
|
function meaningful(s) {
|
|
return s.replace(/[═─━=_#\/*\s|+.-]/g, '').length;
|
|
}
|
|
const MIN_MEANINGFUL = 30;
|
|
|
|
function isRealHeading(h) {
|
|
return !!h && /[A-Za-z0-9]/.test(h.replace(/[═─━=_]/g, ''));
|
|
}
|
|
|
|
function stripPrefix(line, prefix) {
|
|
// '# text' -> 'text' '// text' -> 'text'
|
|
const re = new RegExp('^\\s*' + prefix + '\\s?');
|
|
return line.replace(re, '');
|
|
}
|
|
|
|
// ── Comment-header sectioning, shared by bash (#) and PHP (//) ────────────────────────────────
|
|
function sectionsFromCommentHeader(text, prefix, names) {
|
|
const lines = text.split('\n');
|
|
const nameSet = new Set(names);
|
|
const found = [];
|
|
|
|
// headerEnd matters as much as the section starts. The last section (RUNTIME MODES in bash,
|
|
// DEPENDS ON in a page) would otherwise run to EOF and sweep up every unrelated comment in
|
|
// the file — scheduler.php alone contributed an 11k-char chunk of unrelated inline comments.
|
|
let headerEnd = lines.length;
|
|
for (let i = 0; i < lines.length; i++) {
|
|
const raw = lines[i];
|
|
if (!new RegExp('^\\s*' + prefix).test(raw)) {
|
|
// Header block ends at the first non-comment, non-blank line past the shebang.
|
|
// '<?php' and '?>' bracket a PHP header block and are not the end of it.
|
|
const t = raw.trim();
|
|
if (t !== '' && !/^#!/.test(t) && t !== '<?php' && t !== '?>' && found.length) {
|
|
headerEnd = i;
|
|
break;
|
|
}
|
|
continue;
|
|
}
|
|
const inner = stripPrefix(raw, prefix).trim();
|
|
if (nameSet.has(inner)) found.push({ name: inner, start: i });
|
|
}
|
|
|
|
const out = [];
|
|
for (let k = 0; k < found.length; k++) {
|
|
const start = found[k].start + 1;
|
|
const end = k + 1 < found.length ? found[k + 1].start : headerEnd;
|
|
const body = lines.slice(start, end)
|
|
.filter(l => new RegExp('^\\s*' + prefix).test(l))
|
|
.map(l => stripPrefix(l, prefix))
|
|
// drop pure separator rules (════, ────, ━━━) — they carry no meaning
|
|
.filter(l => !/^[\s═─━=_-]*$/.test(l) || l.trim() === '')
|
|
.join('\n')
|
|
.replace(/\n{3,}/g, '\n\n')
|
|
.trim();
|
|
if (meaningful(body) >= MIN_MEANINGFUL) {
|
|
for (const p of splitNamedParagraphs(body))
|
|
out.push({ section: found[k].name, title: p.title, content: p.content });
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// ── Named-paragraph sub-chunking ──────────────────────────────────────────────────────────────
|
|
// The header convention writes safeguards and principles as named paragraphs: an unindented
|
|
// title line followed by an indented body. Embedding a whole section as one unit dilutes them —
|
|
// rsync.sh's OPERATIONAL SAFEGUARDS holds eight distinct guarantees in 2.8k chars, and a query
|
|
// about one of them scored below unrelated chunks because the other seven dominated the vector.
|
|
// Splitting on the title lines is what makes a specific question find a specific answer.
|
|
//
|
|
// The section name is carried onto every sub-chunk, so section routing still works; the
|
|
// paragraph title becomes the chunk's heading.
|
|
function splitNamedParagraphs(body) {
|
|
const lines = body.split('\n');
|
|
const marks = [];
|
|
for (let i = 0; i < lines.length; i++) {
|
|
const l = lines[i];
|
|
if (!l.trim()) continue;
|
|
if (/^\s/.test(l)) continue; // indented => body, not a title
|
|
if (/^[-*•]/.test(l.trim())) continue; // list item, not a title
|
|
if (l.trim().length > 80) continue; // a long line is prose, not a heading
|
|
if (/[.:;,]$/.test(l.trim())) continue; // ends like a sentence
|
|
|
|
// The decisive signal: a real title is followed by an indented body. Wrapped prose is
|
|
// followed by more unindented prose. Without this check, any short line in a paragraph
|
|
// that happened to wrap became a spurious chunk boundary mid-sentence.
|
|
let j = i + 1;
|
|
while (j < lines.length && !lines[j].trim()) j++;
|
|
if (j >= lines.length || !/^\s+\S/.test(lines[j])) continue;
|
|
|
|
marks.push(i);
|
|
}
|
|
// Fewer than two titles means this section is not written as named paragraphs — keep it whole.
|
|
if (marks.length < 2) return [{ title: null, content: body }];
|
|
|
|
const out = [];
|
|
if (marks[0] > 0) {
|
|
const pre = lines.slice(0, marks[0]).join('\n').trim();
|
|
if (meaningful(pre) >= MIN_MEANINGFUL) out.push({ title: null, content: pre });
|
|
}
|
|
for (let k = 0; k < marks.length; k++) {
|
|
const start = marks[k];
|
|
const end = k + 1 < marks.length ? marks[k + 1] : lines.length;
|
|
const title = lines[start].trim();
|
|
const content = lines.slice(start, end).join('\n').trim();
|
|
if (meaningful(content) >= MIN_MEANINGFUL) out.push({ title, content });
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// ── Markdown: split on ## headings, keep the heading with its body ─────────────────────────────
|
|
function sectionsFromMarkdown(text) {
|
|
const lines = text.split('\n');
|
|
const marks = [];
|
|
let fence = false;
|
|
|
|
for (let i = 0; i < lines.length; i++) {
|
|
if (/^\s*```/.test(lines[i])) { fence = !fence; continue; }
|
|
if (fence) continue;
|
|
if (/^#{1,3}\s+\S/.test(lines[i])) marks.push(i);
|
|
}
|
|
if (!marks.length) return [{ heading: null, content: text.trim() }];
|
|
|
|
const out = [];
|
|
// preamble before the first heading
|
|
if (marks[0] > 0) {
|
|
const pre = lines.slice(0, marks[0]).join('\n').trim();
|
|
if (meaningful(pre) >= MIN_MEANINGFUL) out.push({ heading: null, content: pre });
|
|
}
|
|
for (let k = 0; k < marks.length; k++) {
|
|
const start = marks[k];
|
|
const end = k + 1 < marks.length ? marks[k + 1] : lines.length;
|
|
const heading = lines[start].replace(/^#+\s*/, '').replace(/[━─═]+/g, '').trim();
|
|
const content = lines.slice(start, end).join('\n').trim();
|
|
if (meaningful(content) >= MIN_MEANINGFUL) out.push({ heading: isRealHeading(heading) ? heading : null, content });
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// ── Conf templates: split on the ━━━ / ── section rules ───────────────────────────────────────
|
|
function sectionsFromConfTemplate(text) {
|
|
const lines = text.split('\n');
|
|
const marks = [];
|
|
for (let i = 0; i < lines.length; i++) {
|
|
const m = lines[i].match(/^#\s*[━─]{2,}\s*(.+?)\s*[━─]{2,}\s*$/);
|
|
if (m && isRealHeading(m[1])) marks.push({ i, name: m[1].trim() });
|
|
}
|
|
if (!marks.length) return [];
|
|
|
|
const out = [];
|
|
for (let k = 0; k < marks.length; k++) {
|
|
const start = marks[k].i;
|
|
const end = k + 1 < marks.length ? marks[k + 1].i : lines.length;
|
|
const content = lines.slice(start, end).join('\n').replace(/\n{3,}/g, '\n\n').trim();
|
|
if (meaningful(content) >= MIN_MEANINGFUL) out.push({ heading: marks[k].name, content });
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// Oversized chunks split on blank lines rather than mid-sentence.
|
|
function capSize(chunks) {
|
|
const out = [];
|
|
for (const c of chunks) {
|
|
if (c.content.length <= MAX_CHARS) { out.push(c); continue; }
|
|
const paras = c.content.split(/\n\s*\n/);
|
|
let buf = [], len = 0, part = 1;
|
|
const flush = () => {
|
|
if (!buf.length) return;
|
|
out.push({ ...c, content: buf.join('\n\n'), part: part++ });
|
|
buf = []; len = 0;
|
|
};
|
|
for (const p of paras) {
|
|
if (len + p.length > MAX_CHARS && buf.length) flush();
|
|
buf.push(p); len += p.length + 2;
|
|
}
|
|
flush();
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function classify(rel) {
|
|
const base = path.basename(rel);
|
|
if (rel.startsWith('Deployment/') && rel.endsWith('.template')) return 'template';
|
|
if (base.endsWith('.md')) {
|
|
if (base.startsWith('Manual')) return 'manual';
|
|
if (base.startsWith('README') || base === 'README.md') return 'readme';
|
|
return 'doc';
|
|
}
|
|
if (base.endsWith('.sh')) return 'header';
|
|
if (base.endsWith('.php')) return 'header';
|
|
return 'other';
|
|
}
|
|
|
|
function chunkFile(absPath, rel) {
|
|
const text = fs.readFileSync(absPath, 'utf8');
|
|
const kind = classify(rel);
|
|
let raw = [];
|
|
|
|
if (kind === 'header' && rel.endsWith('.sh')) {
|
|
raw = sectionsFromCommentHeader(text, '#', BASH_SECTIONS)
|
|
.map(s => ({ section: s.section, heading: s.title || null, content: s.content }));
|
|
} else if (kind === 'header' && rel.endsWith('.php')) {
|
|
raw = sectionsFromCommentHeader(text, '//', PHP_SECTIONS)
|
|
.map(s => ({ section: s.section, heading: s.title || null, content: s.content }));
|
|
} else if (kind === 'template') {
|
|
raw = sectionsFromConfTemplate(text)
|
|
.map(s => ({ section: null, heading: s.heading, content: s.content }));
|
|
} else if (kind === 'readme' || kind === 'manual' || kind === 'doc') {
|
|
raw = sectionsFromMarkdown(text)
|
|
.map(s => ({ section: null, heading: s.heading, content: s.content }));
|
|
}
|
|
|
|
return capSize(raw).map(c => ({
|
|
path: rel,
|
|
kind,
|
|
section: c.section || null,
|
|
heading: c.heading || null,
|
|
part: c.part || null,
|
|
content: c.content,
|
|
}));
|
|
}
|
|
|
|
module.exports = { chunkFile, classify, BASH_SECTIONS, PHP_SECTIONS };
|