diff --git a/Plugin/unraid/Tools/ai_chat_worker.php b/Plugin/unraid/Tools/ai_chat_worker.php index 6a47064..bae3f4b 100644 --- a/Plugin/unraid/Tools/ai_chat_worker.php +++ b/Plugin/unraid/Tools/ai_chat_worker.php @@ -50,6 +50,13 @@ // generated answer with no grounding is exactly the confident hallucination this whole // subsystem exists to prevent. // +// Live state is attached only to diagnostic questions. +// Failing health checks and recent log warnings are several thousand tokens. On a +// 16384 context that is budget taken directly from the retrieved passages, so it is +// spent only when the question is asking why something broke. Log lines are marked as +// evidence rather than citable sources, so the model cannot cite a log line as though +// it were documentation. +// // Every failure path writes the job file. // Including the ones that would otherwise be silent — unreachable Ollama, unparseable // response, empty content — so the tab always converges on a state it can render. @@ -116,13 +123,50 @@ foreach ($r['results'] as $i => $x) { $context .= '[' . ($i + 1) . '] ' . $label . "\n" . trim($x['content'] ?? '') . "\n\n"; } +// Diagnostic questions get live state as well as documentation. The docs say what a script is +// supposed to do; only the logs and the current config say what it actually did. Attached only +// when the question is asking why something failed — otherwise it is a few thousand tokens of +// noise competing with the retrieved passages for a context budget that is already tight. +$diagnostic = (bool)preg_match( + '/\b(why|fail(ed|ing|ure)?|error|broken?|not work|isn.t work|wrong|stuck|hang|' + . 'never runs?|didn.t|won.t|debug|troubleshoot|diagnos)/i', + $question +); + +$diagBlock = ''; +if ($diagnostic) { + $bad = array_filter(vv_ai_health(), fn($c) => in_array($c['state'], ['bad', 'warn'], true)); + if ($bad) { + $diagBlock .= "CURRENT CONFIGURATION PROBLEMS\n"; + foreach ($bad as $c) { + $diagBlock .= '- [' . strtoupper($c['state']) . '] ' . $c['label'] . ': ' . $c['detail'] + . ($c['fix'] !== '' ? ' — ' . $c['fix'] : '') . "\n"; + } + $diagBlock .= "\n"; + } + $logs = vv_ai_recent_logs(40); + if ($logs) { + $diagBlock .= "RECENT WARNINGS AND ERRORS (newest last)\n" . implode("\n", $logs) . "\n\n"; + } +} + $system = "You are Varaverk's documentation assistant. Varaverk is this user's private " - . "two-server Unraid media ecosystem; it is not in your training data, so the passages " - . "below are the only thing you know about it.\n\n" - . "Answer only from these passages and cite them inline as [1], [2]. If they do not " - . "contain the answer, say so plainly and name what is missing — do not fill the gap " - . "from general knowledge. Prefer the user's own terminology.\n\n" - . "PASSAGES\n" . $context; + . "two-server Unraid media ecosystem; it is not in your training data, so the material " + . "below is the only thing you know about it.\n\n" + . "Answer only from this material and cite the passages inline as [1], [2]. If it does " + . "not contain the answer, say so plainly and name what is missing — do not fill the " + . "gap from general knowledge. Prefer the user's own terminology.\n\n"; + +if ($diagBlock !== '') { + $system .= "This is a diagnostic question, so live system state is included alongside the " + . "documentation. Use the passages to explain how the thing is supposed to work, " + . "and the live state to say what is actually wrong. When a configuration problem " + . "is listed, name the specific setting and the file it lives in. Log lines are " + . "evidence, not citations — cite only the numbered passages.\n\n" + . $diagBlock; +} + +$system .= "PASSAGES\n" . $context; $messages = [['role' => 'system', 'content' => $system]]; $hist = json_decode($historyJson ?: '[]', true); diff --git a/Plugin/unraid/include/ai.php b/Plugin/unraid/include/ai.php index d9ffad6..3e4a4ce 100644 --- a/Plugin/unraid/include/ai.php +++ b/Plugin/unraid/include/ai.php @@ -29,6 +29,17 @@ // The index only ever contains git-tracked content, so comparing to an untracked scratch // file would report permanent staleness. git ls-files is the same source the indexer uses. // +// Health checks compare configuration against reality, and carry their remedy. +// "Retrieval failed" is not actionable; "HOST1_OLLAMA_MODEL names a tag that is not +// installed" is. Every check names the setting to change, because the failures this +// subsystem actually has are configuration drift — a model pulled, wired into conf, then +// removed — rather than software faults. +// +// Logs are read live, never indexed. +// They churn constantly, would dominate a 3341-chunk corpus, and embedding similarity +// retrieves log lines poorly compared with recency and a severity filter. Indexing them +// would also mean re-embedding every few minutes for content that is stale immediately. +// // OPERATIONAL SAFEGUARDS // Read-only. Nothing here writes to the index, the conf, or the job files — it reads state // and runs a retrieval query. The worker owns every write. @@ -53,7 +64,9 @@ // // EXPORTS // Config vv_ai_enabled(), vv_ai_config() -// Status vv_ai_stats(), vv_ai_index_stats(), vv_ai_runtime_stats() +// Status vv_ai_stats(), vv_ai_index_stats(), vv_ai_runtime_stats(), vv_ai_index_meta() +// Models vv_ai_models_available(), vv_ai_models_loaded() +// Diagnosis vv_ai_health(), vv_ai_recent_logs() // Retrieval vv_ai_retrieve() // Jobs vv_ai_job_dir(), vv_ai_job_path(), vv_ai_job_read() // @@ -178,6 +191,189 @@ function vv_ai_runtime_stats(): array { return $out; } +// Models Ollama actually has on disk. The configured model naming a tag that does not exist is +// the single most likely misconfiguration here — it is what happens whenever a model is pulled, +// wired into conf, and later removed. +function vv_ai_models_available(): ?array { + $cfg = vv_ai_config(); + if ($cfg['url'] === '') return null; + $ctx = stream_context_create(['http' => ['timeout' => max(2, $cfg['connect']), 'ignore_errors' => true]]); + $raw = @file_get_contents($cfg['url'] . '/api/tags', false, $ctx); + if ($raw === false) return null; + $d = json_decode($raw, true); + if (!isset($d['models'])) return null; + return array_map(fn($m) => [ + 'name' => $m['name'] ?? '', + 'size' => (int)($m['size'] ?? 0), + ], $d['models']); +} + +// Currently resident models with their offload split. +function vv_ai_models_loaded(): ?array { + $cfg = vv_ai_config(); + if ($cfg['url'] === '') return null; + $ctx = stream_context_create(['http' => ['timeout' => max(2, $cfg['connect']), 'ignore_errors' => true]]); + $raw = @file_get_contents($cfg['url'] . '/api/ps', false, $ctx); + if ($raw === false) return null; + $d = json_decode($raw, true); + if (!isset($d['models'])) return null; + $out = []; + foreach ($d['models'] as $m) { + $size = (int)($m['size'] ?? 0); + $vram = (int)($m['size_vram'] ?? 0); + $out[] = [ + 'name' => $m['name'] ?? '', + 'context' => $m['context_length'] ?? null, + 'size' => $size, + 'vram' => $vram, + 'offload' => $size > 0 ? (int)round($vram / $size * 100) : null, + ]; + } + return $out; +} + +// vv_meta records the embed model and dimensionality the index was built with. +function vv_ai_index_meta(): array { + $cfg = vv_ai_config(); + if (!file_exists($cfg['db'])) return []; + $raw = trim((string)@shell_exec( + 'sqlite3 ' . escapeshellarg($cfg['db']) . ' ' . escapeshellarg('SELECT key, value FROM vv_meta;') . ' 2>/dev/null' + )); + $out = []; + foreach (explode("\n", $raw) as $line) { + if ($line === '') continue; + [$k, $v] = array_pad(explode('|', $line, 2), 2, ''); + if ($k !== '') $out[$k] = $v; + } + return $out; +} + +// Config checked against reality. Each check is ok | warn | bad, with the remedy attached — +// the point is to name the setting that is wrong, not merely report that something failed. +function vv_ai_health(): array { + $cfg = vv_ai_config(); + $checks = []; + $add = function (string $id, string $label, string $state, string $detail, string $fix = '') + use (&$checks) { + $checks[] = ['id' => $id, 'label' => $label, 'state' => $state, + 'detail' => $detail, 'fix' => $fix]; + }; + + $add('enabled', 'AI enabled', $cfg['enabled'] ? 'ok' : 'bad', + $cfg['enabled'] ? 'AI_ENABLED=true' : 'AI_ENABLED is false', + $cfg['enabled'] ? '' : 'Set AI_ENABLED=true in master.conf'); + + if (!command_exists_node()) { + $add('node', 'node runtime', 'bad', 'node not found on PATH', + 'Retrieval shells to AI/lib/cli.js and cannot run without it'); + } + + if ($cfg['url'] === '') { + $add('url', 'Ollama URL', 'bad', 'not configured', + 'Set ' . strtoupper(vv_detect_host()) . '_OLLAMA_URL in the host conf'); + return $checks; + } + + $avail = vv_ai_models_available(); + if ($avail === null) { + $add('reach', 'Ollama reachable', 'bad', 'no response from ' . $cfg['url'], + 'Check the Ollama container is running and the URL is correct'); + return $checks; + } + $add('reach', 'Ollama reachable', 'ok', $cfg['url']); + + $names = array_column($avail, 'name'); + + // The configured tag not existing is the failure this whole section is for. + if ($cfg['model'] === '') { + $add('gen', 'Generation model', 'bad', 'not configured', + 'Set ' . strtoupper(vv_detect_host()) . '_OLLAMA_MODEL in the host conf'); + } elseif (!in_array($cfg['model'], $names, true)) { + $add('gen', 'Generation model', 'bad', $cfg['model'] . ' is not installed', + 'Either `ollama pull` it, or point _OLLAMA_MODEL at one of: ' . implode(', ', array_slice($names, 0, 4))); + } else { + $add('gen', 'Generation model', 'ok', $cfg['model']); + } + + if ($cfg['embed_model'] === '' || !in_array($cfg['embed_model'], $names, true)) { + $add('embed', 'Embedding model', 'bad', + ($cfg['embed_model'] ?: 'not configured') . ' is not installed', + 'Retrieval cannot embed a query without it — `ollama pull ' . ($cfg['embed_model'] ?: 'nomic-embed-text') . '`'); + } else { + $add('embed', 'Embedding model', 'ok', $cfg['embed_model']); + } + + $ix = vv_ai_index_stats(); + $meta = vv_ai_index_meta(); + + if (!$ix['exists']) { + $add('index', 'Index', 'bad', 'not built', 'Run AI/ai_index.sh'); + } else { + $add('index', 'Index', 'ok', number_format($ix['chunks']) . ' chunks from ' . $ix['files'] . ' files'); + + // A vector built by one embedding model is meaningless to another. Changing the embed + // model without reindexing does not error — it silently returns nonsense, scored + // confidently, which is the hardest failure here to notice from the answers alone. + $builtWith = $meta['embed_model'] ?? ''; + if ($builtWith !== '' && $cfg['embed_model'] !== '' && $builtWith !== $cfg['embed_model']) { + $add('embedmatch', 'Index / embedder match', 'bad', + 'index built with ' . $builtWith . ', conf says ' . $cfg['embed_model'], + 'Vectors from different models are not comparable — rerun AI/ai_index.sh --force'); + } else { + $add('embedmatch', 'Index / embedder match', 'ok', $builtWith ?: 'unrecorded'); + } + + if ($ix['stale'] === true) { + $add('fresh', 'Index freshness', 'warn', 'tracked files are newer than the index', + 'Answers may cite code that has changed — rerun AI/ai_index.sh'); + } else { + $add('fresh', 'Index freshness', 'ok', 'current'); + } + } + + $loaded = vv_ai_models_loaded(); + if ($loaded !== null && $cfg['model'] !== '') { + $hit = null; + foreach ($loaded as $m) if ($m['name'] === $cfg['model']) { $hit = $m; break; } + if ($hit === null) { + $add('resident', 'Model resident', 'warn', 'not loaded — first question will load it'); + } elseif ($hit['offload'] !== null && $hit['offload'] < 100) { + $add('resident', 'GPU offload', 'bad', $hit['offload'] . '% on GPU — layers on CPU', + 'Roughly 4x slower on this card. Lower num_ctx or use a smaller quant'); + } else { + $add('resident', 'GPU offload', 'ok', '100% on GPU at ' . ($hit['context'] ?? '?') . ' ctx'); + } + } + + return $checks; +} + +function command_exists_node(): bool { + static $has = null; + if ($has !== null) return $has; + return $has = trim((string)@shell_exec('command -v node 2>/dev/null')) !== ''; +} + +// Recent warnings and errors across the orchestrator logs. Read live rather than indexed: +// logs churn constantly, would dominate a 3341-chunk index, and embedding similarity retrieves +// them poorly compared with recency plus a severity filter. +function vv_ai_recent_logs(int $max = 40): array { + $dir = '/var/log/varaverk'; + if (!is_dir($dir)) return []; + + $cmd = 'grep -rhE "\[(ERROR|WARN|CRITICAL|FAILED)\]|✗" ' . escapeshellarg($dir) + . ' --include="*.log" 2>/dev/null | tail -' . max(1, min($max, 200)); + $raw = (string)@shell_exec($cmd); + + $out = []; + foreach (explode("\n", $raw) as $line) { + $line = trim(preg_replace('/\033\[[0-9;]*[mK]/', '', $line)); + if ($line === '') continue; + $out[] = mb_substr($line, 0, 300); + } + return $out; +} + function vv_ai_stats(): array { $cfg = vv_ai_config(); return [ @@ -188,6 +384,8 @@ function vv_ai_stats(): array { 'k' => $cfg['k'], 'index' => vv_ai_index_stats(), 'runtime' => vv_ai_runtime_stats(), + 'loaded' => vv_ai_models_loaded(), + 'health' => vv_ai_health(), 'ts' => time(), ]; } diff --git a/Plugin/unraid/pages/ai.php b/Plugin/unraid/pages/ai.php index d5aa44b..f8d7c7c 100644 --- a/Plugin/unraid/pages/ai.php +++ b/Plugin/unraid/pages/ai.php @@ -104,6 +104,23 @@ .vv-ai-src-s { color:#333; font-family:monospace; margin-left:auto; flex-shrink:0; } .vv-ai-meta { font-size:10px; color:#333; margin-top:6px; font-family:monospace; } +/* ── Health + loaded models ─────────────────────────────────────────────── */ +.vv-ai-diag { display:grid; grid-template-columns:1fr 1fr; gap:1px; background:#1a1a1a; + border:1px solid #262626; border-radius:6px; overflow:hidden; } +@media (max-width:900px) { .vv-ai-diag { grid-template-columns:1fr; } } +.vv-ai-diag-col { background:#0e0e0e; padding:9px 12px; } +.vv-ai-diag-h { font-size:9px; letter-spacing:.08em; text-transform:uppercase; color:#4a4a4a; + margin-bottom:6px; display:flex; align-items:center; gap:7px; } +.vv-ai-chk { display:flex; align-items:flex-start; gap:7px; font-size:11px; padding:2px 0; line-height:1.5; } +.vv-ai-chk-i { flex-shrink:0; font-size:11px; width:12px; } +.vv-ai-chk-l { color:#8a8a8a; flex-shrink:0; } +.vv-ai-chk-d { color:#5a5a5a; } +.vv-ai-chk-f { color:#8a6a3a; display:block; font-size:10px; margin-top:1px; } +.vv-ai-mdl { display:flex; align-items:center; gap:7px; font-size:11px; padding:2px 0; } +.vv-ai-mdl-n { color:#8a8a8a; font-family:monospace; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; } +.vv-ai-mdl-m { color:#444; font-family:monospace; margin-left:auto; flex-shrink:0; font-size:10px; } +.vv-ai-none { font-size:11px; color:#3a3a3a; font-style:italic; } + .vv-ai-pending { font-size:12px; color:#5a5a5a; display:flex; align-items:center; gap:8px; } .vv-ai-dot { width:6px; height:6px; border-radius:50%; background:#6fcf97; animation:vvAiPulse 1.1s infinite; } @keyframes vvAiPulse { 0%,100%{opacity:.25;} 50%{opacity:1;} } @@ -141,6 +158,17 @@
+
+
+
System checks
+
+
+
+
Loaded models
+
+
+
+
Ask Varaverk about itself.
@@ -239,6 +267,51 @@ rt.gpu.name + ' · ' + rt.gpu.util + '% util'); } $('vv-ai-banner').innerHTML = html; + renderHealth(s.health || []); + renderModels(s.loaded, s.model); + } + + // ✓ / ! / ✗ per check. Remedy shown inline on anything not ok — the point is to name the + // setting that is wrong, not to report that something failed. + const ICON = { ok: ['✓','vv-ai-ok'], warn: ['!','vv-ai-warn'], bad: ['✗','vv-ai-bad'] }; + function renderHealth(checks) { + if (!checks.length) { $('vv-ai-health').innerHTML = '
no checks
'; return; } + let bad = 0, warn = 0, html = ''; + checks.forEach(c => { + if (c.state === 'bad') bad++; else if (c.state === 'warn') warn++; + const [ic, cl] = ICON[c.state] || ICON.warn; + html += `
${ic}` + + `${esc(c.label)} ` + + `— ${esc(c.detail)}` + + (c.fix ? `→ ${esc(c.fix)}` : '') + + `
`; + }); + $('vv-ai-health').innerHTML = html; + const sum = $('vv-ai-health-sum'); + if (bad) sum.innerHTML = `✗ ${bad} problem${bad>1?'s':''}`; + else if (warn) sum.innerHTML = `! ${warn} warning${warn>1?'s':''}`; + else sum.innerHTML = `✓ all good`; + } + + function renderModels(loaded, active) { + const box = $('vv-ai-models'); + if (loaded === null || loaded === undefined) { + box.innerHTML = '
Ollama unreachable
'; return; + } + if (!loaded.length) { + box.innerHTML = '
none resident — loads on first use
'; return; + } + let html = ''; + loaded.forEach(m => { + const full = m.offload === 100; + const [ic, cl] = full ? ICON.ok : ICON.warn; + html += `
${ic}` + + `` + + esc(m.name.replace(/^hf\.co\/[^/]+\//,'')) + `` + + `${m.offload===null?'—':m.offload+'% GPU'}` + + `${m.context?' · '+m.context.toLocaleString()+' ctx':''} · ${(m.vram/1073741824).toFixed(1)}GB
`; + }); + box.innerHTML = html; } function loadBanner() { fetch(API + '?action=stats').then(r => r.json())