diff --git a/Plugin/unraid/Tools/ai_chat_worker.php b/Plugin/unraid/Tools/ai_chat_worker.php index 440048e..2fff51b 100644 --- a/Plugin/unraid/Tools/ai_chat_worker.php +++ b/Plugin/unraid/Tools/ai_chat_worker.php @@ -70,6 +70,18 @@ // 6 profile varaverk|chat|code|troubleshoot — decides the contract and the inputs // 7 scope what the operator has open, e.g. Orchestrators/daily_sync_maintenance // +// EXPLAIN MODE +// ai_chat_worker.php --explain [--prompt] [profile] [scope] [kind] +// +// Assembles the turn exactly as a real request would, prints which capabilities the profile +// holds, which gates fired, what was attached and what retrieval returned, then exits without +// asking the model. --prompt also dumps the assembled system prompt. +// +// Instant and identical every time, because everything it reports is decided before the model +// is involved. Use it to check a guard rather than reading an answer and inferring one, and to +// tell "the model reasoned badly" apart from "the model was never given the evidence" — which +// look the same from the answer alone. +// // JOB FILE STATES // {"status":"retrieving"} // {"status":"generating","sources":[…]} @@ -85,13 +97,42 @@ if (PHP_SAPI !== 'cli') { require_once dirname(__DIR__) . '/include/ai.php'; -[$jobFile, $question, $historyJson, $kind, $think, $profile, $scope] = - array_slice($argv, 1, 7) + array_fill(0, 7, ''); +// ── explain mode ───────────────────────────────────────────────────────────────────────────── +// Answers "what would this question be given, and why" without asking the model anything. Every +// decision that shapes an answer here is deterministic — which script the question names, which +// capabilities the profile holds, which gates fired, what retrieval returned — and only the prose +// is not. So the half worth testing can be tested without a model call at all: instant, free, and +// identical every time. +// +// It runs the real path rather than describing it. The report is printed from the same variables +// the request uses, immediately before the model call, so it cannot fall out of step with what +// actually happens. A separate function that reconstructed the same decisions would drift within +// a week and then be worse than nothing, because it would be believed. +$explain = ($argv[1] ?? '') === '--explain'; +$showPrompt = false; +if ($explain) { + array_splice($argv, 1, 1); + if (($argv[1] ?? '') === '--prompt') { $showPrompt = true; array_splice($argv, 1, 1); } + [$question, $profile, $scope, $kind] = array_slice($argv, 1, 4) + array_fill(0, 4, ''); + $jobFile = ''; $historyJson = '[]'; $think = '0'; + if ($question === '') { + fwrite(STDERR, "usage: ai_chat_worker.php --explain [--prompt] " + . "[profile] [scope] [kind]\n"); + exit(2); + } +} else { + [$jobFile, $question, $historyJson, $kind, $think, $profile, $scope] = + array_slice($argv, 1, 7) + array_fill(0, 7, ''); -if ($jobFile === '' || $question === '') exit(1); -if (!preg_match('#/[0-9a-f]{32}\.json$#', $jobFile)) exit(1); + if ($jobFile === '' || $question === '') exit(1); + if (!preg_match('#/[0-9a-f]{32}\.json$#', $jobFile)) exit(1); +} + +// What got attached and why, recorded as it happens rather than reconstructed afterwards. +$attached = []; $profile = in_array($profile, ['varaverk', 'chat', 'code', 'troubleshoot'], true) ? $profile : 'varaverk'; +$profileAsked = $profile; // General Chat cannot answer a question about this installation — that is the whole point of it, // and it is why it holds no capabilities. But refusing is not the same as being unable to help, @@ -120,7 +161,10 @@ if ($profile === 'chat' && vv_ai_chat_needs_varaverk($question)) { // definition time would be silently wrong for the rest of the run. $can = function (string $cap) use (&$profile): bool { return vv_ai_profile_can($profile, $cap); }; +// Explain mode has no job file and no tab waiting on one, so the state writes are dropped rather +// than special-cased at each of their call sites. function jw(string $f, array $d): void { + if ($f === '') return; file_put_contents($f, json_encode($d)); } @@ -153,6 +197,7 @@ if ($can('retrieve')) { $r = vv_ai_retrieve($question, $kind); if (!$r['ok']) { + if ($explain) { fwrite(STDERR, 'retrieval failed: ' . ($r['error'] ?? '?') . "\n"); exit(1); } jw($jobFile, ['status' => 'error', 'error' => $r['error'] ?? 'retrieval failed']); exit(1); } @@ -166,7 +211,7 @@ if ($can('retrieve')) { $profile = 'chat'; $escalated = false; wlog('handoff reverted -> chat: nothing in the index for it'); - } else { + } elseif (!$explain) { jw($jobFile, ['status' => 'error', 'error' => 'No relevant documentation found. Try rephrasing, use the readme filter ' . 'for questions about what something is, or switch to General Chat if this ' @@ -234,6 +279,7 @@ if ($diagnostic) { // asked what was wrong, it reported "AI_ENABLED=false" read from the conf *template* // while the live value was true. Documentation records defaults; only this block records // what is actually set. + $attached[] = 'health (' . count(vv_ai_health()) . ' checks)'; $diagBlock .= "LIVE SYSTEM STATE (authoritative — measured just now)\n"; foreach (vv_ai_health() as $c) { $mark = ['ok' => 'OK', 'warn' => 'WARNING', 'bad' => 'PROBLEM'][$c['state']] ?? '?'; @@ -244,6 +290,7 @@ if ($diagnostic) { $logs = vv_ai_recent_logs(40); if ($logs) { + $attached[] = 'recent warnings (' . count($logs) . ' lines)'; $diagBlock .= "RECENT WARNINGS AND ERRORS (newest last)\n" . implode("\n", $logs) . "\n\n"; } } @@ -265,6 +312,8 @@ if ($runTarget !== '' && vv_ai_scope_ok($runTarget)) { // next line, and the difference is the whole answer. $rec = vv_ai_run_record($runTarget); if ($rec['ok']) { + $attached[] = 'run record (' . $rec['status'] + . ', exit ' . var_export($rec['exit'], true) . ')'; $diagBlock .= 'RUN RECORD for ' . $runTarget . " (authoritative — how the last run ended)\n" . '- status: ' . $rec['status'] . ($rec['exit'] !== null ? ' (exit ' . $rec['exit'] . ')' : '') . "\n" @@ -279,10 +328,13 @@ if ($runTarget !== '' && vv_ai_scope_ok($runTarget)) { $scopedLog = vv_ai_scoped_log($runTarget, 120); if ($scopedLog['ok']) { + $attached[] = 'log tail ' . $scopedLog['path'] . ' (' . count($scopedLog['tail']) + . ' of ' . $scopedLog['total'] . ' lines)'; $diagBlock .= 'LOG: ' . $scopedLog['path'] . ' (' . $scopedLog['total'] . " lines total, newest last)\n" . implode("\n", $scopedLog['tail']) . "\n\n"; } else { + $attached[] = 'log tail MISSING for ' . $runTarget . ' (' . ($scopedLog['error'] ?? '?') . ')'; $diagBlock .= "LOG: none found for " . $runTarget . " — it may never have run.\n\n"; } } @@ -296,6 +348,7 @@ if ($runTarget !== '' && vv_ai_scope_ok($runTarget)) { if ($can('incidents') && $scope !== '') { $past = vv_ai_incidents_for($scope, 4); if ($past) { + $attached[] = 'operator incidents (' . count($past) . ')'; $diagBlock .= "PREVIOUSLY ON THIS, WRITTEN BY THE OPERATOR AFTER IT WAS RESOLVED\n" . "Confirmed outcomes, not guesses. If the current symptom matches one of " . "these, say so and lead with it. If it clearly does not, ignore them " @@ -319,6 +372,7 @@ if ($can('conf_lookup')) { } } if ($seen) { + $attached[] = 'conf keys resolved (' . count($seen) . ')'; $diagBlock .= "WHERE THESE SETTINGS ACTUALLY LIVE (looked up just now, authoritative)\n" . implode("\n", $seen) . "\n" . "If this is a different file from the one they have open, say so plainly.\n\n"; @@ -573,6 +627,52 @@ if (is_array($hist)) { } $messages[] = ['role' => 'user', 'content' => $question]; +// Everything above is deterministic. Everything below asks the model. This is the line between +// them, which is why the report prints here: it describes the request that is about to be made, +// not a reconstruction of one. +if ($explain) { + $caps = array_keys(array_filter(VV_AI_CAPS, fn($ps) => in_array($profile, $ps, true))); + $sysChars = strlen($system); + + echo "QUESTION ", $question, "\n"; + echo "PROFILE ", $profileAsked, + $profile !== $profileAsked ? " -> $profile (escalated)" : '', + $escalated ? '' : ($profileAsked === 'chat' && $profile === 'chat' ? ' (no handoff)' : ''), "\n"; + echo "SCOPE ", $scope !== '' ? $scope : '(none)', "\n"; + echo "CAPS ", $caps ? implode(', ', $caps) : '(none — answers from the model alone)', "\n\n"; + + echo "GATES\n"; + printf(" %-14s %s\n", 'named target', $namedTarget !== '' ? $namedTarget : '(none resolved)'); + printf(" %-14s %s\n", 'run outcome', $runOutcome ? 'YES' : 'no'); + printf(" %-14s %s\n", 'diagnostic', $diagnostic ? 'YES' : 'no'); + printf(" %-14s %s\n", 'run target', $runTarget !== '' ? $runTarget : '(none)'); + printf(" %-14s %s\n", 'kind filter', $kind !== '' ? $kind : '(none)'); + echo "\n"; + + echo "ATTACHED\n"; + if (!$attached) { + echo ' (nothing', $sources ? " — no evidence beyond the passages)\n" : ")\n"; + } + foreach ($attached as $a) echo ' ', $a, "\n"; + echo "\n"; + + echo 'RETRIEVED ', count($sources), " passages\n"; + foreach ($sources as $i => $s) { + printf(" [%d] %.3f %s\n", $i + 1, $s['score'], + implode(' › ', array_filter([$s['path'], $s['section'], $s['heading']]))); + } + echo "\n"; + + // The number that actually constrains answer quality. num_ctx is 16384 and the passages are + // the first thing squeezed when evidence blocks grow, so a rough figure here is worth more + // than an exact one later. + printf("PROMPT %d chars, roughly %d tokens of a %d context\n", + $sysChars, (int)round($sysChars / 3.6), 16384); + + if ($showPrompt) echo "\n", str_repeat('─', 92), "\n", $system; + exit(0); +} + $payload = json_encode([ 'model' => $cfg['model'], 'messages' => $messages,