FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Write down the routing every bug so far produced, and check it without asking the model

Browse Source
This commit is contained in:
Gmer4Lfe
2026-08-06 21:34:05 -04:00
parent 2658082607
commit ce0cbd868c
4 changed files with 243 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Plugin/unraid/Tools/ai_chat_worker.php
+12 -10
View File
@@ -129,6 +129,9 @@ if ($explain) {
}
// What got attached and why, recorded as it happens rather than reconstructed afterwards.
// Keyed by a stable token with the varying detail — counts, line totals — kept in the value, so a
// fixture can assert that the log tail was attached without asserting how many lines it had that
// day. An assertion that breaks whenever a log grows is an assertion nobody keeps.
$attached = [];
$profile = in_array($profile, ['varaverk', 'chat', 'code', 'troubleshoot'], true) ? $profile : 'varaverk';
@@ -279,7 +282,7 @@ if ($diagnostic) {
// asked what was wrong, it reported "AI_ENABLED=false" read from the conf *template*
// while the live value was true. Documentation records defaults; only this block records
// what is actually set.
$attached[] = 'health (' . count(vv_ai_health()) . ' checks)';
$attached['health'] = count(vv_ai_health()) . ' checks';
$diagBlock .= "LIVE SYSTEM STATE (authoritative — measured just now)\n";
foreach (vv_ai_health() as $c) {
$mark = ['ok' => 'OK', 'warn' => 'WARNING', 'bad' => 'PROBLEM'][$c['state']] ?? '?';
@@ -290,7 +293,7 @@ if ($diagnostic) {
$logs = vv_ai_recent_logs(40);
if ($logs) {
$attached[] = 'recent warnings (' . count($logs) . ' lines)';
$attached['warnings'] = count($logs) . ' lines';
$diagBlock .= "RECENT WARNINGS AND ERRORS (newest last)\n" . implode("\n", $logs) . "\n\n";
}
}
@@ -312,8 +315,7 @@ if ($runTarget !== '' && vv_ai_scope_ok($runTarget)) {
// next line, and the difference is the whole answer.
$rec = vv_ai_run_record($runTarget);
if ($rec['ok']) {
$attached[] = 'run record (' . $rec['status']
. ', exit ' . var_export($rec['exit'], true) . ')';
$attached['run_record'] = $rec['status'] . ', exit ' . var_export($rec['exit'], true);
$diagBlock .= 'RUN RECORD for ' . $runTarget . " (authoritative — how the last run ended)\n"
. '- status: ' . $rec['status']
. ($rec['exit'] !== null ? ' (exit ' . $rec['exit'] . ')' : '') . "\n"
@@ -328,13 +330,13 @@ if ($runTarget !== '' && vv_ai_scope_ok($runTarget)) {
$scopedLog = vv_ai_scoped_log($runTarget, 120);
if ($scopedLog['ok']) {
$attached[] = 'log tail ' . $scopedLog['path'] . ' (' . count($scopedLog['tail'])
. ' of ' . $scopedLog['total'] . ' lines)';
$attached['log_tail'] = $scopedLog['path'] . ' (' . count($scopedLog['tail'])
. ' of ' . $scopedLog['total'] . ' lines)';
$diagBlock .= 'LOG: ' . $scopedLog['path']
. ' (' . $scopedLog['total'] . " lines total, newest last)\n"
. implode("\n", $scopedLog['tail']) . "\n\n";
} else {
$attached[] = 'log tail MISSING for ' . $runTarget . ' (' . ($scopedLog['error'] ?? '?') . ')';
$attached['log_missing'] = $runTarget . ' — ' . ($scopedLog['error'] ?? '?');
$diagBlock .= "LOG: none found for " . $runTarget . " — it may never have run.\n\n";
}
}
@@ -348,7 +350,7 @@ if ($runTarget !== '' && vv_ai_scope_ok($runTarget)) {
if ($can('incidents') && $scope !== '') {
$past = vv_ai_incidents_for($scope, 4);
if ($past) {
$attached[] = 'operator incidents (' . count($past) . ')';
$attached['incidents'] = (string)count($past);
$diagBlock .= "PREVIOUSLY ON THIS, WRITTEN BY THE OPERATOR AFTER IT WAS RESOLVED\n"
. "Confirmed outcomes, not guesses. If the current symptom matches one of "
. "these, say so and lead with it. If it clearly does not, ignore them "
@@ -372,7 +374,7 @@ if ($can('conf_lookup')) {
}
}
if ($seen) {
$attached[] = 'conf keys resolved (' . count($seen) . ')';
$attached['conf_keys'] = (string)count($seen);
$diagBlock .= "WHERE THESE SETTINGS ACTUALLY LIVE (looked up just now, authoritative)\n"
. implode("\n", $seen) . "\n"
. "If this is a different file from the one they have open, say so plainly.\n\n";
@@ -653,7 +655,7 @@ if ($explain) {
if (!$attached) {
echo ' (nothing', $sources ? " — no evidence beyond the passages)\n" : ")\n";
}
foreach ($attached as $a) echo ' ', $a, "\n";
foreach ($attached as $k => $detail) printf(" %-12s %s\n", $k, $detail);
echo "\n";
echo 'RETRIEVED ', count($sources), " passages\n";
Plugin/unraid/Tools/ai_explain_check.sh
Executable
+158
View File
@@ -0,0 +1,158 @@
#!/bin/bash
# ═══════════════════════════════════════════════════════════════════════════════════════════════
# PURPOSE
# Runs every case in ai_explain_fixtures.txt through the worker's --explain mode and checks the
# routing it reports against what the fixture says it should be. Catches a guard that has been
# undone by a later guard, which is the failure this subsystem keeps producing.
#
# OPERATIONAL MODEL
# Never calls the model. --explain stops on the line where deterministic assembly ends, so a
# full pass costs about a second per case and returns the same answer every time. There is no
# Ollama dependency, no token spend, and no flaky wording to chase.
#
# Not scheduled and deliberately not in any orchestrator. This is a development check — it runs
# when the routing changes, not every night. Nothing on the running system depends on it.
#
# RUNTIME MODES
# ai_explain_check.sh check every fixture
# ai_explain_check.sh --verbose print the full explain report for each case
# ai_explain_check.sh <pattern> only cases whose question matches the pattern
#
# OPERATIONAL SAFEGUARDS
# Asserts routing, never wording.
# Which capabilities a profile holds and which evidence was attached are decided before the
# model is asked anything. Asserting on generated prose would fail for reasons that tell
# nobody anything and the check would be ignored within a fortnight.
#
# A malformed assertion fails loudly rather than passing quietly.
# An unrecognised key is an error, not a skip. A typo in an assertion that silently passes
# is worse than no assertion, because the line still reads as covered.
#
# Exits non-zero on any failure, so it can gate a commit.
#
# DEPENDS ON
# Plugin/unraid/Tools/ai_chat_worker.php --explain mode
# Plugin/unraid/Tools/ai_explain_fixtures.txt
# ═══════════════════════════════════════════════════════════════════════════════════════════════
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
WORKER="$HERE/ai_chat_worker.php"
FIXTURES="$HERE/ai_explain_fixtures.txt"
VERBOSE=false
FILTER=""
for a in "$@"; do
case "$a" in
--verbose) VERBOSE=true ;;
*) FILTER="$a" ;;
esac
done
[[ -f "$WORKER" ]] || { echo "missing worker: $WORKER"; exit 2; }
[[ -f "$FIXTURES" ]] || { echo "missing fixtures: $FIXTURES"; exit 2; }
PASS=0; FAIL=0; SKIP=0
FAILED_LINES=()
trim() { local s="$1"; s="${s#"${s%%[![:space:]]*}"}"; s="${s%"${s##*[![:space:]]}"}"; printf '%s' "$s"; }
# One field out of the explain report. Everything it reads is a fixed label printed by --explain.
field() {
local report="$1" label="$2"
printf '%s' "$report" | grep -m1 -E "^ *$label " | sed -E "s/^ *$label +//" | sed -E 's/ +$//'
}
lineno=0
while IFS= read -r raw || [[ -n "$raw" ]]; do
lineno=$((lineno + 1))
line="$(trim "$raw")"
[[ -z "$line" || "$line" == \#* ]] && continue
IFS='|' read -r q prof scope kind expect <<< "$line"
q="$(trim "$q")"; prof="$(trim "$prof")"; scope="$(trim "$scope")"
kind="$(trim "$kind")"; expect="$(trim "$expect")"
[[ -z "$q" ]] && continue
if [[ -n "$FILTER" && "$q" != *"$FILTER"* ]]; then SKIP=$((SKIP + 1)); continue; fi
report="$(php "$WORKER" --explain "$q" "$prof" "$scope" "$kind" 2>&1)"
if [[ $? -ne 0 ]]; then
echo "✗ line $lineno: explain failed — $q"
printf '%s\n' "$report" | head -3 | sed 's/^/ /'
FAIL=$((FAIL + 1)); FAILED_LINES+=("$lineno"); continue
fi
# PROFILE prints "asked -> used (escalated)" on a handoff and just the profile otherwise.
got_profile="$(printf '%s' "$report" | grep -m1 '^PROFILE' | sed -E 's/^PROFILE +//')"
got_profile="${got_profile##*-> }"
got_profile="$(printf '%s' "$got_profile" | sed -E 's/ *\(.*\)$//')"
got_caps="$(printf '%s' "$report" | grep -m1 '^CAPS' | sed -E 's/^CAPS +//')"
got_target="$(field "$report" 'named target')"
got_run="$(field "$report" 'run outcome')"
got_diag="$(field "$report" 'diagnostic')"
got_attached="$(printf '%s' "$report" | sed -n '/^ATTACHED/,/^$/p' | tail -n +2 \
| awk 'NF {print $1}' | tr '\n' ',' )"
problems=()
for assert in $expect; do
key="${assert%%=*}"; want="${assert#*=}"
case "$key" in
profile)
[[ "$got_profile" == "$want" ]] || problems+=("profile: want $want, got $got_profile") ;;
target)
if [[ "$want" == "none" ]]; then
[[ "$got_target" == "(none resolved)" ]] || problems+=("target: want none, got $got_target")
else
[[ "$got_target" == "$want" ]] || problems+=("target: want $want, got $got_target")
fi ;;
run)
exp=$([[ "$want" == "yes" ]] && echo YES || echo no)
[[ "$got_run" == "$exp" ]] || problems+=("run outcome: want $exp, got $got_run") ;;
diag)
exp=$([[ "$want" == "yes" ]] && echo YES || echo no)
[[ "$got_diag" == "$exp" ]] || problems+=("diagnostic: want $exp, got $got_diag") ;;
caps)
if [[ "$want" == "none" ]]; then
[[ "$got_caps" == "(none"* ]] || problems+=("caps: want none, got $got_caps")
else
[[ "$got_caps" == *"$want"* ]] || problems+=("caps: want $want in [$got_caps]")
fi ;;
has)
IFS=',' read -ra want_keys <<< "$want"
for w in "${want_keys[@]}"; do
[[ ",$got_attached" == *",$w,"* ]] || problems+=("missing attachment: $w")
done ;;
hasnt)
IFS=',' read -ra bad_keys <<< "$want"
for b in "${bad_keys[@]}"; do
[[ ",$got_attached" == *",$b,"* ]] && problems+=("attached but must not be: $b")
done ;;
*)
problems+=("unknown assertion '$key' — typo, or a key this checker does not know") ;;
esac
done
if [[ ${#problems[@]} -eq 0 ]]; then
PASS=$((PASS + 1))
printf '\033[32m✓\033[0m %-6s %s\n' "$prof" "$q"
else
FAIL=$((FAIL + 1)); FAILED_LINES+=("$lineno")
printf '\033[31m✗\033[0m %-6s %s\n' "$prof" "$q"
for p in "${problems[@]}"; do echo " $p"; done
fi
[[ "$VERBOSE" == true ]] && printf '%s\n\n' "$report" | sed 's/^/ /'
done < "$FIXTURES"
echo
echo "─────────────────────────────────────────────"
printf 'passed %d failed %d' "$PASS" "$FAIL"
[[ $SKIP -gt 0 ]] && printf ' skipped %d' "$SKIP"
echo
if [[ $FAIL -gt 0 ]]; then
echo "failing fixture lines: ${FAILED_LINES[*]}"
exit 1
fi
exit 0
Plugin/unraid/Tools/ai_explain_fixtures.txt
+66
View File
@@ -0,0 +1,66 @@
# ═══════════════════════════════════════════════════════════════════════════════════════════════
# Expected routing for the AI assistant, one case per line, checked by ai_explain_check.sh.
#
# Every line here is a bug that was found by reading an answer and noticing it was wrong. That is
# the expensive way to find them and it does not scale: the operator is the test suite, and the
# operator is busy. Written down, each one costs a second to re-check forever.
#
# These assert ROUTING, never wording. What a profile is allowed, which script a question names,
# which gates fire and what evidence gets attached are all decided before the model is involved,
# so they are identical every run. The prose is not and is deliberately not asserted — a test that
# depends on how the model phrases something fails for reasons nobody wants to read about.
#
# FORMAT
# question | profile | scope | kind | assertions
#
# Blank fields are allowed. Assertions are space-separated:
# profile=X the profile that ends up answering, after any handoff
# target=X resolved run target, or 'none'
# run=yes|no the run-outcome gate
# diag=yes|no the diagnostic gate
# caps=none the profile holds no capabilities at all
# has=a,b every one of these must be attached
# hasnt=a,b none of these may be attached
#
# Attachment keys: health warnings run_record log_tail log_missing incidents conf_keys
# ═══════════════════════════════════════════════════════════════════════════════════════════════
# ── Run-outcome questions must arrive with the run, not with directions to the log panel ───────
# The phrasing that started it. "went last" matched; "the run went" did not, one word order apart.
lets check the daily orch log and see how the run went | varaverk | Scheduler | | target=Orchestrators/daily_sync_maintenance run=yes has=run_record,log_tail
lets look at daily orch log and see how it went last run | varaverk | Scheduler | | run=yes has=run_record,log_tail
how did the daily orch go | varaverk | | | target=Orchestrators/daily_sync_maintenance run=yes has=run_record,log_tail
how did the weekly orch go last run | varaverk | | | target=Orchestrators/weekly_sync_maintenance run=yes has=run_record
did the watchdog orchestrator run | varaverk | | | target=Orchestrators/watchdog_orchestrator run=yes
give me a rundown of the daily orch | varaverk | | | run=yes has=run_record
how long did the daily orch take | varaverk | | | run=yes has=run_record
# Works from the AI tab, which sends no scope at all — resolution is from the question, not the page.
how did the daily orch go last run | varaverk | | | target=Orchestrators/daily_sync_maintenance has=run_record
# ── Definitional questions must NOT be answered with last night's log ──────────────────────────
what does the daily orchestrator do | varaverk | | | target=Orchestrators/daily_sync_maintenance run=no hasnt=run_record,log_tail
how does the daily orch work | varaverk | | | run=no hasnt=run_record,log_tail
what is the daily orchestrator | varaverk | | | run=no hasnt=run_record
# ── Ambiguity resolves to nothing rather than guessing a script ────────────────────────────────
# A scored match here attaches the wrong log and answers confidently about a run nobody asked
# about, which is indistinguishable from a right answer unless you already knew.
how did sync go | varaverk | | | target=none run=no
what does RSYNC_ENABLED do | varaverk | | | target=none run=no has=conf_keys
# ── General Chat holds nothing, and hands Varaverk questions up rather than deferring ──────────
how was your day | chat | | | profile=chat caps=none hasnt=health,log_tail,incidents,conf_keys
what does arr_sync.sh do | chat | | | profile=varaverk
is RSYNC_ENABLED on right now | chat | | | profile=varaverk has=conf_keys
how did the daily orch go | chat | | | profile=varaverk run=yes has=run_record
# Chat opened against a script must not be handed the operator's own incident notes about it.
how was your day | chat | Orchestrators/daily_sync_maintenance | | profile=chat hasnt=incidents
# ── Troubleshoot gets the log for whatever is open, plus the right to file a bug ───────────────
why did the weekly orch fail | troubleshoot | Orchestrators/weekly_sync_maintenance | | diag=yes has=log_tail,run_record
what is going on here | troubleshoot | Orchestrators/daily_sync_maintenance | | diag=yes has=log_tail
# ── The code profile answers from the model alone: no passages, no live state ──────────────────
write me a script that copies a folder | code | | | hasnt=health,log_tail,incidents,conf_keys
Plugin/unraid/include/ai.php
+7 -1
View File
@@ -131,7 +131,13 @@ function vv_ai_profile_can(string $profile, string $cap): bool {
// a class of question that is neither answered nor deferred.
function vv_ai_chat_needs_varaverk(string $question): bool {
return vv_ai_mentions_varaverk($question)
|| (bool)preg_match('/\b[\w.-]+\.sh\b|\b[A-Z][A-Z0-9]*(_[A-Z0-9]+)+\b/', $question);
|| (bool)preg_match('/\b[\w.-]+\.sh\b|\b[A-Z][A-Z0-9]*(_[A-Z0-9]+)+\b/', $question)
// Naming something that has a log is as Varaverk-specific as naming a script file, and
// this is how an operator actually refers to them: "the daily orch", not
// "daily_sync_maintenance.sh". Without this, "how did the daily orch go" asked in General
// Chat was refused correctly and then went nowhere — the refusal is right, but the
// question was answerable one profile up and the handoff could not see that.
|| vv_ai_resolve_run_target($question) !== '';
}
function vv_ai_config(): array {