From 237156c46f1f41b558a6571a34db3c378dfa8085 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sun, 2 Aug 2026 17:40:43 -0400 Subject: [PATCH] Add config-vs-reality health checks, loaded models, and log evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The failures this subsystem actually has are configuration drift, so each check names the setting to change rather than reporting that retrieval failed. Notably it catches a conf model tag that is no longer installed, and an index built by a different embedder than the one configured — vectors from two models are not comparable, and that failure returns confident nonsense rather than erroring. Diagnostic questions also get recent log warnings, attached only then because they cost budget the passages need. --- Plugin/unraid/Tools/ai_chat_worker.php | 56 ++++++- Plugin/unraid/include/ai.php | 200 ++++++++++++++++++++++++- Plugin/unraid/pages/ai.php | 73 +++++++++ 3 files changed, 322 insertions(+), 7 deletions(-) diff --git a/Plugin/unraid/Tools/ai_chat_worker.php b/Plugin/unraid/Tools/ai_chat_worker.php index 6a47064..bae3f4b 100644 --- a/Plugin/unraid/Tools/ai_chat_worker.php +++ b/Plugin/unraid/Tools/ai_chat_worker.php @@ -50,6 +50,13 @@ // generated answer with no grounding is exactly the confident hallucination this whole // subsystem exists to prevent. // +// Live state is attached only to diagnostic questions. +// Failing health checks and recent log warnings are several thousand tokens. On a +// 16384 context that is budget taken directly from the retrieved passages, so it is +// spent only when the question is asking why something broke. Log lines are marked as +// evidence rather than citable sources, so the model cannot cite a log line as though +// it were documentation. +// // Every failure path writes the job file. // Including the ones that would otherwise be silent — unreachable Ollama, unparseable // response, empty content — so the tab always converges on a state it can render. @@ -116,13 +123,50 @@ foreach ($r['results'] as $i => $x) { $context .= '[' . ($i + 1) . '] ' . $label . "\n" . trim($x['content'] ?? '') . "\n\n"; } +// Diagnostic questions get live state as well as documentation. The docs say what a script is +// supposed to do; only the logs and the current config say what it actually did. Attached only +// when the question is asking why something failed — otherwise it is a few thousand tokens of +// noise competing with the retrieved passages for a context budget that is already tight. +$diagnostic = (bool)preg_match( + '/\b(why|fail(ed|ing|ure)?|error|broken?|not work|isn.t work|wrong|stuck|hang|' + . 'never runs?|didn.t|won.t|debug|troubleshoot|diagnos)/i', + $question +); + +$diagBlock = ''; +if ($diagnostic) { + $bad = array_filter(vv_ai_health(), fn($c) => in_array($c['state'], ['bad', 'warn'], true)); + if ($bad) { + $diagBlock .= "CURRENT CONFIGURATION PROBLEMS\n"; + foreach ($bad as $c) { + $diagBlock .= '- [' . strtoupper($c['state']) . '] ' . $c['label'] . ': ' . $c['detail'] + . ($c['fix'] !== '' ? ' — ' . $c['fix'] : '') . "\n"; + } + $diagBlock .= "\n"; + } + $logs = vv_ai_recent_logs(40); + if ($logs) { + $diagBlock .= "RECENT WARNINGS AND ERRORS (newest last)\n" . implode("\n", $logs) . "\n\n"; + } +} + $system = "You are Varaverk's documentation assistant. Varaverk is this user's private " - . "two-server Unraid media ecosystem; it is not in your training data, so the passages " - . "below are the only thing you know about it.\n\n" - . "Answer only from these passages and cite them inline as [1], [2]. If they do not " - . "contain the answer, say so plainly and name what is missing — do not fill the gap " - . "from general knowledge. Prefer the user's own terminology.\n\n" - . "PASSAGES\n" . $context; + . "two-server Unraid media ecosystem; it is not in your training data, so the material " + . "below is the only thing you know about it.\n\n" + . "Answer only from this material and cite the passages inline as [1], [2]. If it does " + . "not contain the answer, say so plainly and name what is missing — do not fill the " + . "gap from general knowledge. Prefer the user's own terminology.\n\n"; + +if ($diagBlock !== '') { + $system .= "This is a diagnostic question, so live system state is included alongside the " + . "documentation. Use the passages to explain how the thing is supposed to work, " + . "and the live state to say what is actually wrong. When a configuration problem " + . "is listed, name the specific setting and the file it lives in. Log lines are " + . "evidence, not citations — cite only the numbered passages.\n\n" + . $diagBlock; +} + +$system .= "PASSAGES\n" . $context; $messages = [['role' => 'system', 'content' => $system]]; $hist = json_decode($historyJson ?: '[]', true); diff --git a/Plugin/unraid/include/ai.php b/Plugin/unraid/include/ai.php index d9ffad6..3e4a4ce 100644 --- a/Plugin/unraid/include/ai.php +++ b/Plugin/unraid/include/ai.php @@ -29,6 +29,17 @@ // The index only ever contains git-tracked content, so comparing to an untracked scratch // file would report permanent staleness. git ls-files is the same source the indexer uses. // +// Health checks compare configuration against reality, and carry their remedy. +// "Retrieval failed" is not actionable; "HOST1_OLLAMA_MODEL names a tag that is not +// installed" is. Every check names the setting to change, because the failures this +// subsystem actually has are configuration drift — a model pulled, wired into conf, then +// removed — rather than software faults. +// +// Logs are read live, never indexed. +// They churn constantly, would dominate a 3341-chunk corpus, and embedding similarity +// retrieves log lines poorly compared with recency and a severity filter. Indexing them +// would also mean re-embedding every few minutes for content that is stale immediately. +// // OPERATIONAL SAFEGUARDS // Read-only. Nothing here writes to the index, the conf, or the job files — it reads state // and runs a retrieval query. The worker owns every write. @@ -53,7 +64,9 @@ // // EXPORTS // Config vv_ai_enabled(), vv_ai_config() -// Status vv_ai_stats(), vv_ai_index_stats(), vv_ai_runtime_stats() +// Status vv_ai_stats(), vv_ai_index_stats(), vv_ai_runtime_stats(), vv_ai_index_meta() +// Models vv_ai_models_available(), vv_ai_models_loaded() +// Diagnosis vv_ai_health(), vv_ai_recent_logs() // Retrieval vv_ai_retrieve() // Jobs vv_ai_job_dir(), vv_ai_job_path(), vv_ai_job_read() // @@ -178,6 +191,189 @@ function vv_ai_runtime_stats(): array { return $out; } +// Models Ollama actually has on disk. The configured model naming a tag that does not exist is +// the single most likely misconfiguration here — it is what happens whenever a model is pulled, +// wired into conf, and later removed. +function vv_ai_models_available(): ?array { + $cfg = vv_ai_config(); + if ($cfg['url'] === '') return null; + $ctx = stream_context_create(['http' => ['timeout' => max(2, $cfg['connect']), 'ignore_errors' => true]]); + $raw = @file_get_contents($cfg['url'] . '/api/tags', false, $ctx); + if ($raw === false) return null; + $d = json_decode($raw, true); + if (!isset($d['models'])) return null; + return array_map(fn($m) => [ + 'name' => $m['name'] ?? '', + 'size' => (int)($m['size'] ?? 0), + ], $d['models']); +} + +// Currently resident models with their offload split. +function vv_ai_models_loaded(): ?array { + $cfg = vv_ai_config(); + if ($cfg['url'] === '') return null; + $ctx = stream_context_create(['http' => ['timeout' => max(2, $cfg['connect']), 'ignore_errors' => true]]); + $raw = @file_get_contents($cfg['url'] . '/api/ps', false, $ctx); + if ($raw === false) return null; + $d = json_decode($raw, true); + if (!isset($d['models'])) return null; + $out = []; + foreach ($d['models'] as $m) { + $size = (int)($m['size'] ?? 0); + $vram = (int)($m['size_vram'] ?? 0); + $out[] = [ + 'name' => $m['name'] ?? '', + 'context' => $m['context_length'] ?? null, + 'size' => $size, + 'vram' => $vram, + 'offload' => $size > 0 ? (int)round($vram / $size * 100) : null, + ]; + } + return $out; +} + +// vv_meta records the embed model and dimensionality the index was built with. +function vv_ai_index_meta(): array { + $cfg = vv_ai_config(); + if (!file_exists($cfg['db'])) return []; + $raw = trim((string)@shell_exec( + 'sqlite3 ' . escapeshellarg($cfg['db']) . ' ' . escapeshellarg('SELECT key, value FROM vv_meta;') . ' 2>/dev/null' + )); + $out = []; + foreach (explode("\n", $raw) as $line) { + if ($line === '') continue; + [$k, $v] = array_pad(explode('|', $line, 2), 2, ''); + if ($k !== '') $out[$k] = $v; + } + return $out; +} + +// Config checked against reality. Each check is ok | warn | bad, with the remedy attached — +// the point is to name the setting that is wrong, not merely report that something failed. +function vv_ai_health(): array { + $cfg = vv_ai_config(); + $checks = []; + $add = function (string $id, string $label, string $state, string $detail, string $fix = '') + use (&$checks) { + $checks[] = ['id' => $id, 'label' => $label, 'state' => $state, + 'detail' => $detail, 'fix' => $fix]; + }; + + $add('enabled', 'AI enabled', $cfg['enabled'] ? 'ok' : 'bad', + $cfg['enabled'] ? 'AI_ENABLED=true' : 'AI_ENABLED is false', + $cfg['enabled'] ? '' : 'Set AI_ENABLED=true in master.conf'); + + if (!command_exists_node()) { + $add('node', 'node runtime', 'bad', 'node not found on PATH', + 'Retrieval shells to AI/lib/cli.js and cannot run without it'); + } + + if ($cfg['url'] === '') { + $add('url', 'Ollama URL', 'bad', 'not configured', + 'Set ' . strtoupper(vv_detect_host()) . '_OLLAMA_URL in the host conf'); + return $checks; + } + + $avail = vv_ai_models_available(); + if ($avail === null) { + $add('reach', 'Ollama reachable', 'bad', 'no response from ' . $cfg['url'], + 'Check the Ollama container is running and the URL is correct'); + return $checks; + } + $add('reach', 'Ollama reachable', 'ok', $cfg['url']); + + $names = array_column($avail, 'name'); + + // The configured tag not existing is the failure this whole section is for. + if ($cfg['model'] === '') { + $add('gen', 'Generation model', 'bad', 'not configured', + 'Set ' . strtoupper(vv_detect_host()) . '_OLLAMA_MODEL in the host conf'); + } elseif (!in_array($cfg['model'], $names, true)) { + $add('gen', 'Generation model', 'bad', $cfg['model'] . ' is not installed', + 'Either `ollama pull` it, or point _OLLAMA_MODEL at one of: ' . implode(', ', array_slice($names, 0, 4))); + } else { + $add('gen', 'Generation model', 'ok', $cfg['model']); + } + + if ($cfg['embed_model'] === '' || !in_array($cfg['embed_model'], $names, true)) { + $add('embed', 'Embedding model', 'bad', + ($cfg['embed_model'] ?: 'not configured') . ' is not installed', + 'Retrieval cannot embed a query without it — `ollama pull ' . ($cfg['embed_model'] ?: 'nomic-embed-text') . '`'); + } else { + $add('embed', 'Embedding model', 'ok', $cfg['embed_model']); + } + + $ix = vv_ai_index_stats(); + $meta = vv_ai_index_meta(); + + if (!$ix['exists']) { + $add('index', 'Index', 'bad', 'not built', 'Run AI/ai_index.sh'); + } else { + $add('index', 'Index', 'ok', number_format($ix['chunks']) . ' chunks from ' . $ix['files'] . ' files'); + + // A vector built by one embedding model is meaningless to another. Changing the embed + // model without reindexing does not error — it silently returns nonsense, scored + // confidently, which is the hardest failure here to notice from the answers alone. + $builtWith = $meta['embed_model'] ?? ''; + if ($builtWith !== '' && $cfg['embed_model'] !== '' && $builtWith !== $cfg['embed_model']) { + $add('embedmatch', 'Index / embedder match', 'bad', + 'index built with ' . $builtWith . ', conf says ' . $cfg['embed_model'], + 'Vectors from different models are not comparable — rerun AI/ai_index.sh --force'); + } else { + $add('embedmatch', 'Index / embedder match', 'ok', $builtWith ?: 'unrecorded'); + } + + if ($ix['stale'] === true) { + $add('fresh', 'Index freshness', 'warn', 'tracked files are newer than the index', + 'Answers may cite code that has changed — rerun AI/ai_index.sh'); + } else { + $add('fresh', 'Index freshness', 'ok', 'current'); + } + } + + $loaded = vv_ai_models_loaded(); + if ($loaded !== null && $cfg['model'] !== '') { + $hit = null; + foreach ($loaded as $m) if ($m['name'] === $cfg['model']) { $hit = $m; break; } + if ($hit === null) { + $add('resident', 'Model resident', 'warn', 'not loaded — first question will load it'); + } elseif ($hit['offload'] !== null && $hit['offload'] < 100) { + $add('resident', 'GPU offload', 'bad', $hit['offload'] . '% on GPU — layers on CPU', + 'Roughly 4x slower on this card. Lower num_ctx or use a smaller quant'); + } else { + $add('resident', 'GPU offload', 'ok', '100% on GPU at ' . ($hit['context'] ?? '?') . ' ctx'); + } + } + + return $checks; +} + +function command_exists_node(): bool { + static $has = null; + if ($has !== null) return $has; + return $has = trim((string)@shell_exec('command -v node 2>/dev/null')) !== ''; +} + +// Recent warnings and errors across the orchestrator logs. Read live rather than indexed: +// logs churn constantly, would dominate a 3341-chunk index, and embedding similarity retrieves +// them poorly compared with recency plus a severity filter. +function vv_ai_recent_logs(int $max = 40): array { + $dir = '/var/log/varaverk'; + if (!is_dir($dir)) return []; + + $cmd = 'grep -rhE "\[(ERROR|WARN|CRITICAL|FAILED)\]|✗" ' . escapeshellarg($dir) + . ' --include="*.log" 2>/dev/null | tail -' . max(1, min($max, 200)); + $raw = (string)@shell_exec($cmd); + + $out = []; + foreach (explode("\n", $raw) as $line) { + $line = trim(preg_replace('/\033\[[0-9;]*[mK]/', '', $line)); + if ($line === '') continue; + $out[] = mb_substr($line, 0, 300); + } + return $out; +} + function vv_ai_stats(): array { $cfg = vv_ai_config(); return [ @@ -188,6 +384,8 @@ function vv_ai_stats(): array { 'k' => $cfg['k'], 'index' => vv_ai_index_stats(), 'runtime' => vv_ai_runtime_stats(), + 'loaded' => vv_ai_models_loaded(), + 'health' => vv_ai_health(), 'ts' => time(), ]; } diff --git a/Plugin/unraid/pages/ai.php b/Plugin/unraid/pages/ai.php index d5aa44b..f8d7c7c 100644 --- a/Plugin/unraid/pages/ai.php +++ b/Plugin/unraid/pages/ai.php @@ -104,6 +104,23 @@ .vv-ai-src-s { color:#333; font-family:monospace; margin-left:auto; flex-shrink:0; } .vv-ai-meta { font-size:10px; color:#333; margin-top:6px; font-family:monospace; } +/* ── Health + loaded models ─────────────────────────────────────────────── */ +.vv-ai-diag { display:grid; grid-template-columns:1fr 1fr; gap:1px; background:#1a1a1a; + border:1px solid #262626; border-radius:6px; overflow:hidden; } +@media (max-width:900px) { .vv-ai-diag { grid-template-columns:1fr; } } +.vv-ai-diag-col { background:#0e0e0e; padding:9px 12px; } +.vv-ai-diag-h { font-size:9px; letter-spacing:.08em; text-transform:uppercase; color:#4a4a4a; + margin-bottom:6px; display:flex; align-items:center; gap:7px; } +.vv-ai-chk { display:flex; align-items:flex-start; gap:7px; font-size:11px; padding:2px 0; line-height:1.5; } +.vv-ai-chk-i { flex-shrink:0; font-size:11px; width:12px; } +.vv-ai-chk-l { color:#8a8a8a; flex-shrink:0; } +.vv-ai-chk-d { color:#5a5a5a; } +.vv-ai-chk-f { color:#8a6a3a; display:block; font-size:10px; margin-top:1px; } +.vv-ai-mdl { display:flex; align-items:center; gap:7px; font-size:11px; padding:2px 0; } +.vv-ai-mdl-n { color:#8a8a8a; font-family:monospace; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; } +.vv-ai-mdl-m { color:#444; font-family:monospace; margin-left:auto; flex-shrink:0; font-size:10px; } +.vv-ai-none { font-size:11px; color:#3a3a3a; font-style:italic; } + .vv-ai-pending { font-size:12px; color:#5a5a5a; display:flex; align-items:center; gap:8px; } .vv-ai-dot { width:6px; height:6px; border-radius:50%; background:#6fcf97; animation:vvAiPulse 1.1s infinite; } @keyframes vvAiPulse { 0%,100%{opacity:.25;} 50%{opacity:1;} } @@ -141,6 +158,17 @@
+
+
+
System checks
+
+
+
+
Loaded models
+
+
+
+
Ask Varaverk about itself.
@@ -239,6 +267,51 @@ rt.gpu.name + ' · ' + rt.gpu.util + '% util'); } $('vv-ai-banner').innerHTML = html; + renderHealth(s.health || []); + renderModels(s.loaded, s.model); + } + + // ✓ / ! / ✗ per check. Remedy shown inline on anything not ok — the point is to name the + // setting that is wrong, not to report that something failed. + const ICON = { ok: ['✓','vv-ai-ok'], warn: ['!','vv-ai-warn'], bad: ['✗','vv-ai-bad'] }; + function renderHealth(checks) { + if (!checks.length) { $('vv-ai-health').innerHTML = '
no checks
'; return; } + let bad = 0, warn = 0, html = ''; + checks.forEach(c => { + if (c.state === 'bad') bad++; else if (c.state === 'warn') warn++; + const [ic, cl] = ICON[c.state] || ICON.warn; + html += `
${ic}` + + `${esc(c.label)} ` + + `— ${esc(c.detail)}` + + (c.fix ? `→ ${esc(c.fix)}` : '') + + `
`; + }); + $('vv-ai-health').innerHTML = html; + const sum = $('vv-ai-health-sum'); + if (bad) sum.innerHTML = `✗ ${bad} problem${bad>1?'s':''}`; + else if (warn) sum.innerHTML = `! ${warn} warning${warn>1?'s':''}`; + else sum.innerHTML = `✓ all good`; + } + + function renderModels(loaded, active) { + const box = $('vv-ai-models'); + if (loaded === null || loaded === undefined) { + box.innerHTML = '
Ollama unreachable
'; return; + } + if (!loaded.length) { + box.innerHTML = '
none resident — loads on first use
'; return; + } + let html = ''; + loaded.forEach(m => { + const full = m.offload === 100; + const [ic, cl] = full ? ICON.ok : ICON.warn; + html += `
${ic}` + + `` + + esc(m.name.replace(/^hf\.co\/[^/]+\//,'')) + `` + + `${m.offload===null?'—':m.offload+'% GPU'}` + + `${m.context?' · '+m.context.toLocaleString()+' ctx':''} · ${(m.vram/1073741824).toFixed(1)}GB
`; + }); + box.innerHTML = html; } function loadBanner() { fetch(API + '?action=stats').then(r => r.json())