From 269ef9b2de8ee01ee4d3da1625acc78e005b1e76 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Wed, 5 Aug 2026 21:25:00 -0400 Subject: [PATCH] Verify a bug report's component and evidence before filing it --- Plugin/unraid/Tools/ai_chat_worker.php | 48 ++++++++++++++++++++++---- Plugin/unraid/include/ai.php | 36 +++++++++++++++++++ Plugin/unraid/pages/ai.php | 4 ++- 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/Plugin/unraid/Tools/ai_chat_worker.php b/Plugin/unraid/Tools/ai_chat_worker.php index bdd17e9..bc8457c 100644 --- a/Plugin/unraid/Tools/ai_chat_worker.php +++ b/Plugin/unraid/Tools/ai_chat_worker.php @@ -94,6 +94,14 @@ function jw(string $f, array $d): void { file_put_contents($f, json_encode($d)); } +// Same log the endpoint writes to, tagged so the two are tellable apart. Defined here because +// vv_ai_log() belongs to api/ai.php and this runs detached, with no endpoint in the process. +function wlog(string $msg): void { + if (!is_dir('/var/log/varaverk')) return; + @file_put_contents('/var/log/varaverk/ai.log', + date('Y-m-d H:i:s') . ' worker ' . $msg . "\n", FILE_APPEND | LOCK_EX); +} + $cfg = vv_ai_config(); $t0 = microtime(true); @@ -489,12 +497,40 @@ if ($profile === 'troubleshoot' // evidence may run to several lines; take everything after the label. $ev = preg_match('/^\s*evidence\s*:\s*(.*)$/mis', $bm[1], $em) ? trim($em[1]) : ''; - $res = vv_ai_bug_write($field('component'), $field('summary'), $ev, [ - 'asked' => mb_substr($question, 0, 300), - 'scope' => $scope, - 'log' => $scopedLog['path'] ?? null, - 'profile' => $profile, - ]); + $component = $field('component'); + + // Guard one: the component must be a real file here. A report against a path that does not + // exist is the model naming something plausible rather than something it saw. + // + // Guard two: the quoted evidence must actually appear in the log it was given. This is what + // keeps "misconfigured" out of the bug list — a genuine defect is visible in the log, while + // a setting being false produces no such line, so an invented quote fails here instead of + // becoming a report someone has to disprove. + // + // When no log was available there is nothing to check against; the report is still filed, + // because live health state is real evidence too, but it is marked unverified and says so + // on the card. Refusals are logged rather than swallowed — how often the model tries to file + // junk is worth knowing, and silence would hide it. + $verified = $scopedLog && !empty($scopedLog['ok']) + ? vv_ai_evidence_in_log($ev, $scopedLog['tail'] ?? []) + : null; + + if (!vv_ai_component_exists($component)) { + wlog('bug refused: component not a real file — ' . mb_substr($component, 0, 80)); + $res = ['ok' => false]; + } elseif ($verified === false) { + wlog('bug refused: evidence not found in ' . ($scopedLog['path'] ?? '?') + . ' — ' . mb_substr(preg_replace('/\s+/', ' ', $ev), 0, 100)); + $res = ['ok' => false]; + } else { + $res = vv_ai_bug_write($component, $field('summary'), $ev, [ + 'asked' => mb_substr($question, 0, 300), + 'scope' => $scope, + 'log' => $scopedLog['path'] ?? null, + 'verified' => $verified, + 'profile' => $profile, + ]); + } if ($res['ok']) { $bugFiled = $res; // Say what actually happened, not what sounds reassuring. This is written to a file on diff --git a/Plugin/unraid/include/ai.php b/Plugin/unraid/include/ai.php index c01b692..11825ac 100644 --- a/Plugin/unraid/include/ai.php +++ b/Plugin/unraid/include/ai.php @@ -815,6 +815,42 @@ function vv_ai_bug_file(array $r): string { return vv_ai_bugs_dir() . '/' . substr(sha1(strtolower($r['component'] . '|' . $r['summary'])), 0, 12) . '.json'; } +// Does the quoted evidence actually appear in the log the model was shown? +// +// The prompt asks for a verbatim line. This checks it, because "manufactured a cause to have +// one" is the specific failure this profile was warned against and warnings are not guards — +// the same reason General Chat needed a regex behind its instruction, not just a firmer wording. +// +// Whitespace-normalised and matched on the longest quoted fragment: the model reliably keeps the +// text and unreliably keeps the indentation. A short fragment would match by coincidence, so +// anything under 25 characters is not treated as a match at all. +function vv_ai_evidence_in_log(string $evidence, array $tail): bool { + if (!$tail) return false; + $norm = fn(string $s) => preg_replace('/\s+/', ' ', trim($s)); + $hay = ' ' . implode(' ⏎ ', array_map($norm, $tail)) . ' '; + + $frags = array_filter(array_map($norm, preg_split('/\R/', $evidence)), + fn($l) => mb_strlen($l) >= 25); + usort($frags, fn($a, $b) => mb_strlen($b) <=> mb_strlen($a)); + foreach (array_slice($frags, 0, 5) as $f) { + if (str_contains($hay, $f)) return true; + // Allow a trimmed tail of the fragment — models often drop a trailing clause. + $head = mb_substr($f, 0, max(25, (int)(mb_strlen($f) * 0.6))); + if (mb_strlen($head) >= 25 && str_contains($hay, $head)) return true; + } + return false; +} + +// Is the named component a real file in this installation? A defect report against a path that +// does not exist is a confabulation, and it is free to check. +function vv_ai_component_exists(string $component): bool { + $c = trim($component); + if ($c === '' || !vv_ai_scope_ok($c)) return false; + $base = realpath(SCRIPTS_DIR); + $p = realpath(SCRIPTS_DIR . '/' . $c); + return $base !== false && $p !== false && str_starts_with($p, $base . '/') && is_file($p); +} + function vv_ai_bug_write(string $component, string $summary, string $evidence, array $ctx = []): array { $component = trim($component); $summary = trim($summary); diff --git a/Plugin/unraid/pages/ai.php b/Plugin/unraid/pages/ai.php index 7dd2a1d..6d94efc 100644 --- a/Plugin/unraid/pages/ai.php +++ b/Plugin/unraid/pages/ai.php @@ -551,7 +551,9 @@ if (is_dir('/var/log/varaverk')) { return `
${esc(b.component)} - ${esc(b.id)}${esc(seen)} · ${esc(when)} + ${esc(b.id)}${esc(seen)} · ${esc(when)}${ + b.context && b.context.verified === null + ? ' · evidence unverified' : ''}