diff --git a/Plugin/unraid/Tools/ai_chat_worker.php b/Plugin/unraid/Tools/ai_chat_worker.php index 2fff51b..a153e56 100644 --- a/Plugin/unraid/Tools/ai_chat_worker.php +++ b/Plugin/unraid/Tools/ai_chat_worker.php @@ -129,6 +129,9 @@ if ($explain) { } // What got attached and why, recorded as it happens rather than reconstructed afterwards. +// Keyed by a stable token with the varying detail — counts, line totals — kept in the value, so a +// fixture can assert that the log tail was attached without asserting how many lines it had that +// day. An assertion that breaks whenever a log grows is an assertion nobody keeps. $attached = []; $profile = in_array($profile, ['varaverk', 'chat', 'code', 'troubleshoot'], true) ? $profile : 'varaverk'; @@ -279,7 +282,7 @@ if ($diagnostic) { // asked what was wrong, it reported "AI_ENABLED=false" read from the conf *template* // while the live value was true. Documentation records defaults; only this block records // what is actually set. - $attached[] = 'health (' . count(vv_ai_health()) . ' checks)'; + $attached['health'] = count(vv_ai_health()) . ' checks'; $diagBlock .= "LIVE SYSTEM STATE (authoritative — measured just now)\n"; foreach (vv_ai_health() as $c) { $mark = ['ok' => 'OK', 'warn' => 'WARNING', 'bad' => 'PROBLEM'][$c['state']] ?? '?'; @@ -290,7 +293,7 @@ if ($diagnostic) { $logs = vv_ai_recent_logs(40); if ($logs) { - $attached[] = 'recent warnings (' . count($logs) . ' lines)'; + $attached['warnings'] = count($logs) . ' lines'; $diagBlock .= "RECENT WARNINGS AND ERRORS (newest last)\n" . implode("\n", $logs) . "\n\n"; } } @@ -312,8 +315,7 @@ if ($runTarget !== '' && vv_ai_scope_ok($runTarget)) { // next line, and the difference is the whole answer. $rec = vv_ai_run_record($runTarget); if ($rec['ok']) { - $attached[] = 'run record (' . $rec['status'] - . ', exit ' . var_export($rec['exit'], true) . ')'; + $attached['run_record'] = $rec['status'] . ', exit ' . var_export($rec['exit'], true); $diagBlock .= 'RUN RECORD for ' . $runTarget . " (authoritative — how the last run ended)\n" . '- status: ' . $rec['status'] . ($rec['exit'] !== null ? ' (exit ' . $rec['exit'] . ')' : '') . "\n" @@ -328,13 +330,13 @@ if ($runTarget !== '' && vv_ai_scope_ok($runTarget)) { $scopedLog = vv_ai_scoped_log($runTarget, 120); if ($scopedLog['ok']) { - $attached[] = 'log tail ' . $scopedLog['path'] . ' (' . count($scopedLog['tail']) - . ' of ' . $scopedLog['total'] . ' lines)'; + $attached['log_tail'] = $scopedLog['path'] . ' (' . count($scopedLog['tail']) + . ' of ' . $scopedLog['total'] . ' lines)'; $diagBlock .= 'LOG: ' . $scopedLog['path'] . ' (' . $scopedLog['total'] . " lines total, newest last)\n" . implode("\n", $scopedLog['tail']) . "\n\n"; } else { - $attached[] = 'log tail MISSING for ' . $runTarget . ' (' . ($scopedLog['error'] ?? '?') . ')'; + $attached['log_missing'] = $runTarget . ' — ' . ($scopedLog['error'] ?? '?'); $diagBlock .= "LOG: none found for " . $runTarget . " — it may never have run.\n\n"; } } @@ -348,7 +350,7 @@ if ($runTarget !== '' && vv_ai_scope_ok($runTarget)) { if ($can('incidents') && $scope !== '') { $past = vv_ai_incidents_for($scope, 4); if ($past) { - $attached[] = 'operator incidents (' . count($past) . ')'; + $attached['incidents'] = (string)count($past); $diagBlock .= "PREVIOUSLY ON THIS, WRITTEN BY THE OPERATOR AFTER IT WAS RESOLVED\n" . "Confirmed outcomes, not guesses. If the current symptom matches one of " . "these, say so and lead with it. If it clearly does not, ignore them " @@ -372,7 +374,7 @@ if ($can('conf_lookup')) { } } if ($seen) { - $attached[] = 'conf keys resolved (' . count($seen) . ')'; + $attached['conf_keys'] = (string)count($seen); $diagBlock .= "WHERE THESE SETTINGS ACTUALLY LIVE (looked up just now, authoritative)\n" . implode("\n", $seen) . "\n" . "If this is a different file from the one they have open, say so plainly.\n\n"; @@ -653,7 +655,7 @@ if ($explain) { if (!$attached) { echo ' (nothing', $sources ? " — no evidence beyond the passages)\n" : ")\n"; } - foreach ($attached as $a) echo ' ', $a, "\n"; + foreach ($attached as $k => $detail) printf(" %-12s %s\n", $k, $detail); echo "\n"; echo 'RETRIEVED ', count($sources), " passages\n"; diff --git a/Plugin/unraid/Tools/ai_explain_check.sh b/Plugin/unraid/Tools/ai_explain_check.sh new file mode 100755 index 0000000..01554a6 --- /dev/null +++ b/Plugin/unraid/Tools/ai_explain_check.sh @@ -0,0 +1,158 @@ +#!/bin/bash +# ═══════════════════════════════════════════════════════════════════════════════════════════════ +# PURPOSE +# Runs every case in ai_explain_fixtures.txt through the worker's --explain mode and checks the +# routing it reports against what the fixture says it should be. Catches a guard that has been +# undone by a later guard, which is the failure this subsystem keeps producing. +# +# OPERATIONAL MODEL +# Never calls the model. --explain stops on the line where deterministic assembly ends, so a +# full pass costs about a second per case and returns the same answer every time. There is no +# Ollama dependency, no token spend, and no flaky wording to chase. +# +# Not scheduled and deliberately not in any orchestrator. This is a development check — it runs +# when the routing changes, not every night. Nothing on the running system depends on it. +# +# RUNTIME MODES +# ai_explain_check.sh check every fixture +# ai_explain_check.sh --verbose print the full explain report for each case +# ai_explain_check.sh only cases whose question matches the pattern +# +# OPERATIONAL SAFEGUARDS +# Asserts routing, never wording. +# Which capabilities a profile holds and which evidence was attached are decided before the +# model is asked anything. Asserting on generated prose would fail for reasons that tell +# nobody anything and the check would be ignored within a fortnight. +# +# A malformed assertion fails loudly rather than passing quietly. +# An unrecognised key is an error, not a skip. A typo in an assertion that silently passes +# is worse than no assertion, because the line still reads as covered. +# +# Exits non-zero on any failure, so it can gate a commit. +# +# DEPENDS ON +# Plugin/unraid/Tools/ai_chat_worker.php --explain mode +# Plugin/unraid/Tools/ai_explain_fixtures.txt +# ═══════════════════════════════════════════════════════════════════════════════════════════════ + +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORKER="$HERE/ai_chat_worker.php" +FIXTURES="$HERE/ai_explain_fixtures.txt" + +VERBOSE=false +FILTER="" +for a in "$@"; do + case "$a" in + --verbose) VERBOSE=true ;; + *) FILTER="$a" ;; + esac +done + +[[ -f "$WORKER" ]] || { echo "missing worker: $WORKER"; exit 2; } +[[ -f "$FIXTURES" ]] || { echo "missing fixtures: $FIXTURES"; exit 2; } + +PASS=0; FAIL=0; SKIP=0 +FAILED_LINES=() + +trim() { local s="$1"; s="${s#"${s%%[![:space:]]*}"}"; s="${s%"${s##*[![:space:]]}"}"; printf '%s' "$s"; } + +# One field out of the explain report. Everything it reads is a fixed label printed by --explain. +field() { + local report="$1" label="$2" + printf '%s' "$report" | grep -m1 -E "^ *$label " | sed -E "s/^ *$label +//" | sed -E 's/ +$//' +} + +lineno=0 +while IFS= read -r raw || [[ -n "$raw" ]]; do + lineno=$((lineno + 1)) + line="$(trim "$raw")" + [[ -z "$line" || "$line" == \#* ]] && continue + + IFS='|' read -r q prof scope kind expect <<< "$line" + q="$(trim "$q")"; prof="$(trim "$prof")"; scope="$(trim "$scope")" + kind="$(trim "$kind")"; expect="$(trim "$expect")" + [[ -z "$q" ]] && continue + + if [[ -n "$FILTER" && "$q" != *"$FILTER"* ]]; then SKIP=$((SKIP + 1)); continue; fi + + report="$(php "$WORKER" --explain "$q" "$prof" "$scope" "$kind" 2>&1)" + if [[ $? -ne 0 ]]; then + echo "✗ line $lineno: explain failed — $q" + printf '%s\n' "$report" | head -3 | sed 's/^/ /' + FAIL=$((FAIL + 1)); FAILED_LINES+=("$lineno"); continue + fi + + # PROFILE prints "asked -> used (escalated)" on a handoff and just the profile otherwise. + got_profile="$(printf '%s' "$report" | grep -m1 '^PROFILE' | sed -E 's/^PROFILE +//')" + got_profile="${got_profile##*-> }" + got_profile="$(printf '%s' "$got_profile" | sed -E 's/ *\(.*\)$//')" + got_caps="$(printf '%s' "$report" | grep -m1 '^CAPS' | sed -E 's/^CAPS +//')" + got_target="$(field "$report" 'named target')" + got_run="$(field "$report" 'run outcome')" + got_diag="$(field "$report" 'diagnostic')" + got_attached="$(printf '%s' "$report" | sed -n '/^ATTACHED/,/^$/p' | tail -n +2 \ + | awk 'NF {print $1}' | tr '\n' ',' )" + + problems=() + for assert in $expect; do + key="${assert%%=*}"; want="${assert#*=}" + case "$key" in + profile) + [[ "$got_profile" == "$want" ]] || problems+=("profile: want $want, got $got_profile") ;; + target) + if [[ "$want" == "none" ]]; then + [[ "$got_target" == "(none resolved)" ]] || problems+=("target: want none, got $got_target") + else + [[ "$got_target" == "$want" ]] || problems+=("target: want $want, got $got_target") + fi ;; + run) + exp=$([[ "$want" == "yes" ]] && echo YES || echo no) + [[ "$got_run" == "$exp" ]] || problems+=("run outcome: want $exp, got $got_run") ;; + diag) + exp=$([[ "$want" == "yes" ]] && echo YES || echo no) + [[ "$got_diag" == "$exp" ]] || problems+=("diagnostic: want $exp, got $got_diag") ;; + caps) + if [[ "$want" == "none" ]]; then + [[ "$got_caps" == "(none"* ]] || problems+=("caps: want none, got $got_caps") + else + [[ "$got_caps" == *"$want"* ]] || problems+=("caps: want $want in [$got_caps]") + fi ;; + has) + IFS=',' read -ra want_keys <<< "$want" + for w in "${want_keys[@]}"; do + [[ ",$got_attached" == *",$w,"* ]] || problems+=("missing attachment: $w") + done ;; + hasnt) + IFS=',' read -ra bad_keys <<< "$want" + for b in "${bad_keys[@]}"; do + [[ ",$got_attached" == *",$b,"* ]] && problems+=("attached but must not be: $b") + done ;; + *) + problems+=("unknown assertion '$key' — typo, or a key this checker does not know") ;; + esac + done + + if [[ ${#problems[@]} -eq 0 ]]; then + PASS=$((PASS + 1)) + printf '\033[32m✓\033[0m %-6s %s\n' "$prof" "$q" + else + FAIL=$((FAIL + 1)); FAILED_LINES+=("$lineno") + printf '\033[31m✗\033[0m %-6s %s\n' "$prof" "$q" + for p in "${problems[@]}"; do echo " $p"; done + fi + + [[ "$VERBOSE" == true ]] && printf '%s\n\n' "$report" | sed 's/^/ /' +done < "$FIXTURES" + +echo +echo "─────────────────────────────────────────────" +printf 'passed %d failed %d' "$PASS" "$FAIL" +[[ $SKIP -gt 0 ]] && printf ' skipped %d' "$SKIP" +echo +if [[ $FAIL -gt 0 ]]; then + echo "failing fixture lines: ${FAILED_LINES[*]}" + exit 1 +fi +exit 0 diff --git a/Plugin/unraid/Tools/ai_explain_fixtures.txt b/Plugin/unraid/Tools/ai_explain_fixtures.txt new file mode 100644 index 0000000..d22bd83 --- /dev/null +++ b/Plugin/unraid/Tools/ai_explain_fixtures.txt @@ -0,0 +1,66 @@ +# ═══════════════════════════════════════════════════════════════════════════════════════════════ +# Expected routing for the AI assistant, one case per line, checked by ai_explain_check.sh. +# +# Every line here is a bug that was found by reading an answer and noticing it was wrong. That is +# the expensive way to find them and it does not scale: the operator is the test suite, and the +# operator is busy. Written down, each one costs a second to re-check forever. +# +# These assert ROUTING, never wording. What a profile is allowed, which script a question names, +# which gates fire and what evidence gets attached are all decided before the model is involved, +# so they are identical every run. The prose is not and is deliberately not asserted — a test that +# depends on how the model phrases something fails for reasons nobody wants to read about. +# +# FORMAT +# question | profile | scope | kind | assertions +# +# Blank fields are allowed. Assertions are space-separated: +# profile=X the profile that ends up answering, after any handoff +# target=X resolved run target, or 'none' +# run=yes|no the run-outcome gate +# diag=yes|no the diagnostic gate +# caps=none the profile holds no capabilities at all +# has=a,b every one of these must be attached +# hasnt=a,b none of these may be attached +# +# Attachment keys: health warnings run_record log_tail log_missing incidents conf_keys +# ═══════════════════════════════════════════════════════════════════════════════════════════════ + +# ── Run-outcome questions must arrive with the run, not with directions to the log panel ─────── +# The phrasing that started it. "went last" matched; "the run went" did not, one word order apart. +lets check the daily orch log and see how the run went | varaverk | Scheduler | | target=Orchestrators/daily_sync_maintenance run=yes has=run_record,log_tail +lets look at daily orch log and see how it went last run | varaverk | Scheduler | | run=yes has=run_record,log_tail +how did the daily orch go | varaverk | | | target=Orchestrators/daily_sync_maintenance run=yes has=run_record,log_tail +how did the weekly orch go last run | varaverk | | | target=Orchestrators/weekly_sync_maintenance run=yes has=run_record +did the watchdog orchestrator run | varaverk | | | target=Orchestrators/watchdog_orchestrator run=yes +give me a rundown of the daily orch | varaverk | | | run=yes has=run_record +how long did the daily orch take | varaverk | | | run=yes has=run_record + +# Works from the AI tab, which sends no scope at all — resolution is from the question, not the page. +how did the daily orch go last run | varaverk | | | target=Orchestrators/daily_sync_maintenance has=run_record + +# ── Definitional questions must NOT be answered with last night's log ────────────────────────── +what does the daily orchestrator do | varaverk | | | target=Orchestrators/daily_sync_maintenance run=no hasnt=run_record,log_tail +how does the daily orch work | varaverk | | | run=no hasnt=run_record,log_tail +what is the daily orchestrator | varaverk | | | run=no hasnt=run_record + +# ── Ambiguity resolves to nothing rather than guessing a script ──────────────────────────────── +# A scored match here attaches the wrong log and answers confidently about a run nobody asked +# about, which is indistinguishable from a right answer unless you already knew. +how did sync go | varaverk | | | target=none run=no +what does RSYNC_ENABLED do | varaverk | | | target=none run=no has=conf_keys + +# ── General Chat holds nothing, and hands Varaverk questions up rather than deferring ────────── +how was your day | chat | | | profile=chat caps=none hasnt=health,log_tail,incidents,conf_keys +what does arr_sync.sh do | chat | | | profile=varaverk +is RSYNC_ENABLED on right now | chat | | | profile=varaverk has=conf_keys +how did the daily orch go | chat | | | profile=varaverk run=yes has=run_record + +# Chat opened against a script must not be handed the operator's own incident notes about it. +how was your day | chat | Orchestrators/daily_sync_maintenance | | profile=chat hasnt=incidents + +# ── Troubleshoot gets the log for whatever is open, plus the right to file a bug ─────────────── +why did the weekly orch fail | troubleshoot | Orchestrators/weekly_sync_maintenance | | diag=yes has=log_tail,run_record +what is going on here | troubleshoot | Orchestrators/daily_sync_maintenance | | diag=yes has=log_tail + +# ── The code profile answers from the model alone: no passages, no live state ────────────────── +write me a script that copies a folder | code | | | hasnt=health,log_tail,incidents,conf_keys diff --git a/Plugin/unraid/include/ai.php b/Plugin/unraid/include/ai.php index fb5d801..db045fa 100644 --- a/Plugin/unraid/include/ai.php +++ b/Plugin/unraid/include/ai.php @@ -131,7 +131,13 @@ function vv_ai_profile_can(string $profile, string $cap): bool { // a class of question that is neither answered nor deferred. function vv_ai_chat_needs_varaverk(string $question): bool { return vv_ai_mentions_varaverk($question) - || (bool)preg_match('/\b[\w.-]+\.sh\b|\b[A-Z][A-Z0-9]*(_[A-Z0-9]+)+\b/', $question); + || (bool)preg_match('/\b[\w.-]+\.sh\b|\b[A-Z][A-Z0-9]*(_[A-Z0-9]+)+\b/', $question) + // Naming something that has a log is as Varaverk-specific as naming a script file, and + // this is how an operator actually refers to them: "the daily orch", not + // "daily_sync_maintenance.sh". Without this, "how did the daily orch go" asked in General + // Chat was refused correctly and then went nowhere — the refusal is right, but the + // question was answerable one profile up and the handoff could not see that. + || vv_ai_resolve_run_target($question) !== ''; } function vv_ai_config(): array {