Write down the routing every bug so far produced, and check it without asking the model
This commit is contained in:
Gmer4Lfe
@@ -129,6 +129,9 @@ if ($explain) {
|
||||
}
|
||||
|
||||
// What got attached and why, recorded as it happens rather than reconstructed afterwards.
|
||||
// Keyed by a stable token with the varying detail — counts, line totals — kept in the value, so a
|
||||
// fixture can assert that the log tail was attached without asserting how many lines it had that
|
||||
// day. An assertion that breaks whenever a log grows is an assertion nobody keeps.
|
||||
$attached = [];
|
||||
|
||||
$profile = in_array($profile, ['varaverk', 'chat', 'code', 'troubleshoot'], true) ? $profile : 'varaverk';
|
||||
@@ -279,7 +282,7 @@ if ($diagnostic) {
|
||||
// asked what was wrong, it reported "AI_ENABLED=false" read from the conf *template*
|
||||
// while the live value was true. Documentation records defaults; only this block records
|
||||
// what is actually set.
|
||||
$attached[] = 'health (' . count(vv_ai_health()) . ' checks)';
|
||||
$attached['health'] = count(vv_ai_health()) . ' checks';
|
||||
$diagBlock .= "LIVE SYSTEM STATE (authoritative — measured just now)\n";
|
||||
foreach (vv_ai_health() as $c) {
|
||||
$mark = ['ok' => 'OK', 'warn' => 'WARNING', 'bad' => 'PROBLEM'][$c['state']] ?? '?';
|
||||
@@ -290,7 +293,7 @@ if ($diagnostic) {
|
||||
|
||||
$logs = vv_ai_recent_logs(40);
|
||||
if ($logs) {
|
||||
$attached[] = 'recent warnings (' . count($logs) . ' lines)';
|
||||
$attached['warnings'] = count($logs) . ' lines';
|
||||
$diagBlock .= "RECENT WARNINGS AND ERRORS (newest last)\n" . implode("\n", $logs) . "\n\n";
|
||||
}
|
||||
}
|
||||
@@ -312,8 +315,7 @@ if ($runTarget !== '' && vv_ai_scope_ok($runTarget)) {
|
||||
// next line, and the difference is the whole answer.
|
||||
$rec = vv_ai_run_record($runTarget);
|
||||
if ($rec['ok']) {
|
||||
$attached[] = 'run record (' . $rec['status']
|
||||
. ', exit ' . var_export($rec['exit'], true) . ')';
|
||||
$attached['run_record'] = $rec['status'] . ', exit ' . var_export($rec['exit'], true);
|
||||
$diagBlock .= 'RUN RECORD for ' . $runTarget . " (authoritative — how the last run ended)\n"
|
||||
. '- status: ' . $rec['status']
|
||||
. ($rec['exit'] !== null ? ' (exit ' . $rec['exit'] . ')' : '') . "\n"
|
||||
@@ -328,13 +330,13 @@ if ($runTarget !== '' && vv_ai_scope_ok($runTarget)) {
|
||||
|
||||
$scopedLog = vv_ai_scoped_log($runTarget, 120);
|
||||
if ($scopedLog['ok']) {
|
||||
$attached[] = 'log tail ' . $scopedLog['path'] . ' (' . count($scopedLog['tail'])
|
||||
. ' of ' . $scopedLog['total'] . ' lines)';
|
||||
$attached['log_tail'] = $scopedLog['path'] . ' (' . count($scopedLog['tail'])
|
||||
. ' of ' . $scopedLog['total'] . ' lines)';
|
||||
$diagBlock .= 'LOG: ' . $scopedLog['path']
|
||||
. ' (' . $scopedLog['total'] . " lines total, newest last)\n"
|
||||
. implode("\n", $scopedLog['tail']) . "\n\n";
|
||||
} else {
|
||||
$attached[] = 'log tail MISSING for ' . $runTarget . ' (' . ($scopedLog['error'] ?? '?') . ')';
|
||||
$attached['log_missing'] = $runTarget . ' — ' . ($scopedLog['error'] ?? '?');
|
||||
$diagBlock .= "LOG: none found for " . $runTarget . " — it may never have run.\n\n";
|
||||
}
|
||||
}
|
||||
@@ -348,7 +350,7 @@ if ($runTarget !== '' && vv_ai_scope_ok($runTarget)) {
|
||||
if ($can('incidents') && $scope !== '') {
|
||||
$past = vv_ai_incidents_for($scope, 4);
|
||||
if ($past) {
|
||||
$attached[] = 'operator incidents (' . count($past) . ')';
|
||||
$attached['incidents'] = (string)count($past);
|
||||
$diagBlock .= "PREVIOUSLY ON THIS, WRITTEN BY THE OPERATOR AFTER IT WAS RESOLVED\n"
|
||||
. "Confirmed outcomes, not guesses. If the current symptom matches one of "
|
||||
. "these, say so and lead with it. If it clearly does not, ignore them "
|
||||
@@ -372,7 +374,7 @@ if ($can('conf_lookup')) {
|
||||
}
|
||||
}
|
||||
if ($seen) {
|
||||
$attached[] = 'conf keys resolved (' . count($seen) . ')';
|
||||
$attached['conf_keys'] = (string)count($seen);
|
||||
$diagBlock .= "WHERE THESE SETTINGS ACTUALLY LIVE (looked up just now, authoritative)\n"
|
||||
. implode("\n", $seen) . "\n"
|
||||
. "If this is a different file from the one they have open, say so plainly.\n\n";
|
||||
@@ -653,7 +655,7 @@ if ($explain) {
|
||||
if (!$attached) {
|
||||
echo ' (nothing', $sources ? " — no evidence beyond the passages)\n" : ")\n";
|
||||
}
|
||||
foreach ($attached as $a) echo ' ', $a, "\n";
|
||||
foreach ($attached as $k => $detail) printf(" %-12s %s\n", $k, $detail);
|
||||
echo "\n";
|
||||
|
||||
echo 'RETRIEVED ', count($sources), " passages\n";
|
||||
|
||||
Executable
+158
@@ -0,0 +1,158 @@
|
||||
#!/bin/bash
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# PURPOSE
|
||||
# Runs every case in ai_explain_fixtures.txt through the worker's --explain mode and checks the
|
||||
# routing it reports against what the fixture says it should be. Catches a guard that has been
|
||||
# undone by a later guard, which is the failure this subsystem keeps producing.
|
||||
#
|
||||
# OPERATIONAL MODEL
|
||||
# Never calls the model. --explain stops on the line where deterministic assembly ends, so a
|
||||
# full pass costs about a second per case and returns the same answer every time. There is no
|
||||
# Ollama dependency, no token spend, and no flaky wording to chase.
|
||||
#
|
||||
# Not scheduled and deliberately not in any orchestrator. This is a development check — it runs
|
||||
# when the routing changes, not every night. Nothing on the running system depends on it.
|
||||
#
|
||||
# RUNTIME MODES
|
||||
# ai_explain_check.sh check every fixture
|
||||
# ai_explain_check.sh --verbose print the full explain report for each case
|
||||
# ai_explain_check.sh <pattern> only cases whose question matches the pattern
|
||||
#
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# Asserts routing, never wording.
|
||||
# Which capabilities a profile holds and which evidence was attached are decided before the
|
||||
# model is asked anything. Asserting on generated prose would fail for reasons that tell
|
||||
# nobody anything and the check would be ignored within a fortnight.
|
||||
#
|
||||
# A malformed assertion fails loudly rather than passing quietly.
|
||||
# An unrecognised key is an error, not a skip. A typo in an assertion that silently passes
|
||||
# is worse than no assertion, because the line still reads as covered.
|
||||
#
|
||||
# Exits non-zero on any failure, so it can gate a commit.
|
||||
#
|
||||
# DEPENDS ON
|
||||
# Plugin/unraid/Tools/ai_chat_worker.php --explain mode
|
||||
# Plugin/unraid/Tools/ai_explain_fixtures.txt
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
WORKER="$HERE/ai_chat_worker.php"
|
||||
FIXTURES="$HERE/ai_explain_fixtures.txt"
|
||||
|
||||
VERBOSE=false
|
||||
FILTER=""
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
--verbose) VERBOSE=true ;;
|
||||
*) FILTER="$a" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -f "$WORKER" ]] || { echo "missing worker: $WORKER"; exit 2; }
|
||||
[[ -f "$FIXTURES" ]] || { echo "missing fixtures: $FIXTURES"; exit 2; }
|
||||
|
||||
PASS=0; FAIL=0; SKIP=0
|
||||
FAILED_LINES=()
|
||||
|
||||
trim() { local s="$1"; s="${s#"${s%%[![:space:]]*}"}"; s="${s%"${s##*[![:space:]]}"}"; printf '%s' "$s"; }
|
||||
|
||||
# One field out of the explain report. Everything it reads is a fixed label printed by --explain.
|
||||
field() {
|
||||
local report="$1" label="$2"
|
||||
printf '%s' "$report" | grep -m1 -E "^ *$label " | sed -E "s/^ *$label +//" | sed -E 's/ +$//'
|
||||
}
|
||||
|
||||
lineno=0
|
||||
while IFS= read -r raw || [[ -n "$raw" ]]; do
|
||||
lineno=$((lineno + 1))
|
||||
line="$(trim "$raw")"
|
||||
[[ -z "$line" || "$line" == \#* ]] && continue
|
||||
|
||||
IFS='|' read -r q prof scope kind expect <<< "$line"
|
||||
q="$(trim "$q")"; prof="$(trim "$prof")"; scope="$(trim "$scope")"
|
||||
kind="$(trim "$kind")"; expect="$(trim "$expect")"
|
||||
[[ -z "$q" ]] && continue
|
||||
|
||||
if [[ -n "$FILTER" && "$q" != *"$FILTER"* ]]; then SKIP=$((SKIP + 1)); continue; fi
|
||||
|
||||
report="$(php "$WORKER" --explain "$q" "$prof" "$scope" "$kind" 2>&1)"
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "✗ line $lineno: explain failed — $q"
|
||||
printf '%s\n' "$report" | head -3 | sed 's/^/ /'
|
||||
FAIL=$((FAIL + 1)); FAILED_LINES+=("$lineno"); continue
|
||||
fi
|
||||
|
||||
# PROFILE prints "asked -> used (escalated)" on a handoff and just the profile otherwise.
|
||||
got_profile="$(printf '%s' "$report" | grep -m1 '^PROFILE' | sed -E 's/^PROFILE +//')"
|
||||
got_profile="${got_profile##*-> }"
|
||||
got_profile="$(printf '%s' "$got_profile" | sed -E 's/ *\(.*\)$//')"
|
||||
got_caps="$(printf '%s' "$report" | grep -m1 '^CAPS' | sed -E 's/^CAPS +//')"
|
||||
got_target="$(field "$report" 'named target')"
|
||||
got_run="$(field "$report" 'run outcome')"
|
||||
got_diag="$(field "$report" 'diagnostic')"
|
||||
got_attached="$(printf '%s' "$report" | sed -n '/^ATTACHED/,/^$/p' | tail -n +2 \
|
||||
| awk 'NF {print $1}' | tr '\n' ',' )"
|
||||
|
||||
problems=()
|
||||
for assert in $expect; do
|
||||
key="${assert%%=*}"; want="${assert#*=}"
|
||||
case "$key" in
|
||||
profile)
|
||||
[[ "$got_profile" == "$want" ]] || problems+=("profile: want $want, got $got_profile") ;;
|
||||
target)
|
||||
if [[ "$want" == "none" ]]; then
|
||||
[[ "$got_target" == "(none resolved)" ]] || problems+=("target: want none, got $got_target")
|
||||
else
|
||||
[[ "$got_target" == "$want" ]] || problems+=("target: want $want, got $got_target")
|
||||
fi ;;
|
||||
run)
|
||||
exp=$([[ "$want" == "yes" ]] && echo YES || echo no)
|
||||
[[ "$got_run" == "$exp" ]] || problems+=("run outcome: want $exp, got $got_run") ;;
|
||||
diag)
|
||||
exp=$([[ "$want" == "yes" ]] && echo YES || echo no)
|
||||
[[ "$got_diag" == "$exp" ]] || problems+=("diagnostic: want $exp, got $got_diag") ;;
|
||||
caps)
|
||||
if [[ "$want" == "none" ]]; then
|
||||
[[ "$got_caps" == "(none"* ]] || problems+=("caps: want none, got $got_caps")
|
||||
else
|
||||
[[ "$got_caps" == *"$want"* ]] || problems+=("caps: want $want in [$got_caps]")
|
||||
fi ;;
|
||||
has)
|
||||
IFS=',' read -ra want_keys <<< "$want"
|
||||
for w in "${want_keys[@]}"; do
|
||||
[[ ",$got_attached" == *",$w,"* ]] || problems+=("missing attachment: $w")
|
||||
done ;;
|
||||
hasnt)
|
||||
IFS=',' read -ra bad_keys <<< "$want"
|
||||
for b in "${bad_keys[@]}"; do
|
||||
[[ ",$got_attached" == *",$b,"* ]] && problems+=("attached but must not be: $b")
|
||||
done ;;
|
||||
*)
|
||||
problems+=("unknown assertion '$key' — typo, or a key this checker does not know") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ ${#problems[@]} -eq 0 ]]; then
|
||||
PASS=$((PASS + 1))
|
||||
printf '\033[32m✓\033[0m %-6s %s\n' "$prof" "$q"
|
||||
else
|
||||
FAIL=$((FAIL + 1)); FAILED_LINES+=("$lineno")
|
||||
printf '\033[31m✗\033[0m %-6s %s\n' "$prof" "$q"
|
||||
for p in "${problems[@]}"; do echo " $p"; done
|
||||
fi
|
||||
|
||||
[[ "$VERBOSE" == true ]] && printf '%s\n\n' "$report" | sed 's/^/ /'
|
||||
done < "$FIXTURES"
|
||||
|
||||
echo
|
||||
echo "─────────────────────────────────────────────"
|
||||
printf 'passed %d failed %d' "$PASS" "$FAIL"
|
||||
[[ $SKIP -gt 0 ]] && printf ' skipped %d' "$SKIP"
|
||||
echo
|
||||
if [[ $FAIL -gt 0 ]]; then
|
||||
echo "failing fixture lines: ${FAILED_LINES[*]}"
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
@@ -0,0 +1,66 @@
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# Expected routing for the AI assistant, one case per line, checked by ai_explain_check.sh.
|
||||
#
|
||||
# Every line here is a bug that was found by reading an answer and noticing it was wrong. That is
|
||||
# the expensive way to find them and it does not scale: the operator is the test suite, and the
|
||||
# operator is busy. Written down, each one costs a second to re-check forever.
|
||||
#
|
||||
# These assert ROUTING, never wording. What a profile is allowed, which script a question names,
|
||||
# which gates fire and what evidence gets attached are all decided before the model is involved,
|
||||
# so they are identical every run. The prose is not and is deliberately not asserted — a test that
|
||||
# depends on how the model phrases something fails for reasons nobody wants to read about.
|
||||
#
|
||||
# FORMAT
|
||||
# question | profile | scope | kind | assertions
|
||||
#
|
||||
# Blank fields are allowed. Assertions are space-separated:
|
||||
# profile=X the profile that ends up answering, after any handoff
|
||||
# target=X resolved run target, or 'none'
|
||||
# run=yes|no the run-outcome gate
|
||||
# diag=yes|no the diagnostic gate
|
||||
# caps=none the profile holds no capabilities at all
|
||||
# has=a,b every one of these must be attached
|
||||
# hasnt=a,b none of these may be attached
|
||||
#
|
||||
# Attachment keys: health warnings run_record log_tail log_missing incidents conf_keys
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
# ── Run-outcome questions must arrive with the run, not with directions to the log panel ───────
|
||||
# The phrasing that started it. "went last" matched; "the run went" did not, one word order apart.
|
||||
lets check the daily orch log and see how the run went | varaverk | Scheduler | | target=Orchestrators/daily_sync_maintenance run=yes has=run_record,log_tail
|
||||
lets look at daily orch log and see how it went last run | varaverk | Scheduler | | run=yes has=run_record,log_tail
|
||||
how did the daily orch go | varaverk | | | target=Orchestrators/daily_sync_maintenance run=yes has=run_record,log_tail
|
||||
how did the weekly orch go last run | varaverk | | | target=Orchestrators/weekly_sync_maintenance run=yes has=run_record
|
||||
did the watchdog orchestrator run | varaverk | | | target=Orchestrators/watchdog_orchestrator run=yes
|
||||
give me a rundown of the daily orch | varaverk | | | run=yes has=run_record
|
||||
how long did the daily orch take | varaverk | | | run=yes has=run_record
|
||||
|
||||
# Works from the AI tab, which sends no scope at all — resolution is from the question, not the page.
|
||||
how did the daily orch go last run | varaverk | | | target=Orchestrators/daily_sync_maintenance has=run_record
|
||||
|
||||
# ── Definitional questions must NOT be answered with last night's log ──────────────────────────
|
||||
what does the daily orchestrator do | varaverk | | | target=Orchestrators/daily_sync_maintenance run=no hasnt=run_record,log_tail
|
||||
how does the daily orch work | varaverk | | | run=no hasnt=run_record,log_tail
|
||||
what is the daily orchestrator | varaverk | | | run=no hasnt=run_record
|
||||
|
||||
# ── Ambiguity resolves to nothing rather than guessing a script ────────────────────────────────
|
||||
# A scored match here attaches the wrong log and answers confidently about a run nobody asked
|
||||
# about, which is indistinguishable from a right answer unless you already knew.
|
||||
how did sync go | varaverk | | | target=none run=no
|
||||
what does RSYNC_ENABLED do | varaverk | | | target=none run=no has=conf_keys
|
||||
|
||||
# ── General Chat holds nothing, and hands Varaverk questions up rather than deferring ──────────
|
||||
how was your day | chat | | | profile=chat caps=none hasnt=health,log_tail,incidents,conf_keys
|
||||
what does arr_sync.sh do | chat | | | profile=varaverk
|
||||
is RSYNC_ENABLED on right now | chat | | | profile=varaverk has=conf_keys
|
||||
how did the daily orch go | chat | | | profile=varaverk run=yes has=run_record
|
||||
|
||||
# Chat opened against a script must not be handed the operator's own incident notes about it.
|
||||
how was your day | chat | Orchestrators/daily_sync_maintenance | | profile=chat hasnt=incidents
|
||||
|
||||
# ── Troubleshoot gets the log for whatever is open, plus the right to file a bug ───────────────
|
||||
why did the weekly orch fail | troubleshoot | Orchestrators/weekly_sync_maintenance | | diag=yes has=log_tail,run_record
|
||||
what is going on here | troubleshoot | Orchestrators/daily_sync_maintenance | | diag=yes has=log_tail
|
||||
|
||||
# ── The code profile answers from the model alone: no passages, no live state ──────────────────
|
||||
write me a script that copies a folder | code | | | hasnt=health,log_tail,incidents,conf_keys
|
||||
Reference in New Issue
Block a user