Gmer4Lfe
613634473a
Search is the one capability that fits the profile holding none: every other capability either reads this installation or changes it, and chat's whole contract is that it can do neither. The assistant deliberately does not get it — its contract is that answers come from this install's own documents, and a web result there is an answer that looks sourced and is not. A chat question about this machine hands off to the assistant before the search would run, so it never reaches the internet even with the box ticked. Off by default, and not because it is dangerous: searching sends the operator's question outside the house, which is theirs to decide. Asked for per turn as well as enabled in conf. Provider-agnostic, as asked — searxng, brave, tavily. Only whichever is configured here can be verified; all three read every field by name, so a shape that changes underneath yields no results rather than wrong ones. The explain fixtures asserted chat holds no capabilities at all, which is exactly the guarantee worth keeping. caps=only: now states the set rather than its emptiness, so the check still fails the day something else is granted there.
175 lines
8.0 KiB
Bash
Executable File
175 lines
8.0 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# =============================== AI Explain Check =============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# Runs every case in ai_explain_fixtures.txt through the worker's --explain mode and checks the
|
|
# routing it reports against what the fixture says it should be. Catches a guard that has been
|
|
# undone by a later guard, which is the failure this subsystem keeps producing.
|
|
#
|
|
# OPERATIONAL MODEL
|
|
# Never calls the model. --explain stops on the line where deterministic assembly ends, so a
|
|
# full pass costs about a second per case and returns the same answer every time. There is no
|
|
# Ollama dependency, no token spend, and no flaky wording to chase.
|
|
#
|
|
# Not scheduled and deliberately not in any orchestrator. This is a development check — it runs
|
|
# when the routing changes, not every night. Nothing on the running system depends on it.
|
|
#
|
|
# RUNTIME MODES
|
|
# ai_explain_check.sh check every fixture
|
|
# ai_explain_check.sh --verbose print the full explain report for each case
|
|
# ai_explain_check.sh <pattern> only cases whose question matches the pattern
|
|
#
|
|
# OPERATIONAL SAFEGUARDS
|
|
# Asserts routing, never wording.
|
|
# Which capabilities a profile holds and which evidence was attached are decided before the
|
|
# model is asked anything. Asserting on generated prose would fail for reasons that tell
|
|
# nobody anything and the check would be ignored within a fortnight.
|
|
#
|
|
# A malformed assertion fails loudly rather than passing quietly.
|
|
# An unrecognised key is an error, not a skip. A typo in an assertion that silently passes
|
|
# is worse than no assertion, because the line still reads as covered.
|
|
#
|
|
# Exits non-zero on any failure, so it can gate a commit.
|
|
#
|
|
# DEPENDS ON
|
|
# Plugin/unraid/Tools/ai_chat_worker.php --explain mode
|
|
# Plugin/unraid/Tools/ai_explain_fixtures.txt
|
|
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
set -uo pipefail
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
WORKER="$HERE/ai_chat_worker.php"
|
|
FIXTURES="$HERE/ai_explain_fixtures.txt"
|
|
|
|
VERBOSE=false
|
|
FILTER=""
|
|
for a in "$@"; do
|
|
case "$a" in
|
|
--verbose) VERBOSE=true ;;
|
|
*) FILTER="$a" ;;
|
|
esac
|
|
done
|
|
|
|
[[ -f "$WORKER" ]] || { echo "missing worker: $WORKER"; exit 2; }
|
|
[[ -f "$FIXTURES" ]] || { echo "missing fixtures: $FIXTURES"; exit 2; }
|
|
|
|
PASS=0; FAIL=0; SKIP=0
|
|
FAILED_LINES=()
|
|
|
|
trim() { local s="$1"; s="${s#"${s%%[![:space:]]*}"}"; s="${s%"${s##*[![:space:]]}"}"; printf '%s' "$s"; }
|
|
|
|
# One field out of the explain report. Everything it reads is a fixed label printed by --explain.
|
|
field() {
|
|
local report="$1" label="$2"
|
|
printf '%s' "$report" | grep -m1 -E "^ *$label " | sed -E "s/^ *$label +//" | sed -E 's/ +$//'
|
|
}
|
|
|
|
lineno=0
|
|
while IFS= read -r raw || [[ -n "$raw" ]]; do
|
|
lineno=$((lineno + 1))
|
|
line="$(trim "$raw")"
|
|
[[ -z "$line" || "$line" == \#* ]] && continue
|
|
|
|
IFS='|' read -r q prof scope kind expect <<< "$line"
|
|
q="$(trim "$q")"; prof="$(trim "$prof")"; scope="$(trim "$scope")"
|
|
kind="$(trim "$kind")"; expect="$(trim "$expect")"
|
|
[[ -z "$q" ]] && continue
|
|
|
|
if [[ -n "$FILTER" && "$q" != *"$FILTER"* ]]; then SKIP=$((SKIP + 1)); continue; fi
|
|
|
|
report="$(php "$WORKER" --explain "$q" "$prof" "$scope" "$kind" 2>&1)"
|
|
if [[ $? -ne 0 ]]; then
|
|
echo "✗ line $lineno: explain failed — $q"
|
|
printf '%s\n' "$report" | head -3 | sed 's/^/ /'
|
|
FAIL=$((FAIL + 1)); FAILED_LINES+=("$lineno"); continue
|
|
fi
|
|
|
|
# PROFILE prints "asked -> used (escalated)" on a handoff and just the profile otherwise.
|
|
got_profile="$(printf '%s' "$report" | grep -m1 '^PROFILE' | sed -E 's/^PROFILE +//')"
|
|
got_profile="${got_profile##*-> }"
|
|
got_profile="$(printf '%s' "$got_profile" | sed -E 's/ *\(.*\)$//')"
|
|
got_caps="$(printf '%s' "$report" | grep -m1 '^CAPS' | sed -E 's/^CAPS +//')"
|
|
got_target="$(field "$report" 'named target')"
|
|
got_run="$(field "$report" 'run outcome')"
|
|
got_diag="$(field "$report" 'diagnostic')"
|
|
got_attached="$(printf '%s' "$report" | sed -n '/^ATTACHED/,/^$/p' | tail -n +2 \
|
|
| awk 'NF {print $1}' | tr '\n' ',' )"
|
|
|
|
problems=()
|
|
for assert in $expect; do
|
|
key="${assert%%=*}"; want="${assert#*=}"
|
|
case "$key" in
|
|
profile)
|
|
[[ "$got_profile" == "$want" ]] || problems+=("profile: want $want, got $got_profile") ;;
|
|
target)
|
|
if [[ "$want" == "none" ]]; then
|
|
[[ "$got_target" == "(none resolved)" ]] || problems+=("target: want none, got $got_target")
|
|
else
|
|
[[ "$got_target" == "$want" ]] || problems+=("target: want $want, got $got_target")
|
|
fi ;;
|
|
run)
|
|
exp=$([[ "$want" == "yes" ]] && echo YES || echo no)
|
|
[[ "$got_run" == "$exp" ]] || problems+=("run outcome: want $exp, got $got_run") ;;
|
|
diag)
|
|
exp=$([[ "$want" == "yes" ]] && echo YES || echo no)
|
|
[[ "$got_diag" == "$exp" ]] || problems+=("diagnostic: want $exp, got $got_diag") ;;
|
|
caps)
|
|
# none holds nothing at all
|
|
# only:a,b holds exactly these and nothing else
|
|
# a holds at least this
|
|
#
|
|
# only: exists because General Chat's guarantee is not "it can search" but "search
|
|
# is the only thing it can do". A presence check would still pass on the day
|
|
# something else is granted there, which is the day the check was written for.
|
|
if [[ "$want" == "none" ]]; then
|
|
[[ "$got_caps" == "(none"* ]] || problems+=("caps: want none, got $got_caps")
|
|
elif [[ "$want" == only:* ]]; then
|
|
want_set="${want#only:}"
|
|
got_set="$(printf '%s' "$got_caps" | tr -d ' ' | tr ',' '\n' | sort | paste -sd, -)"
|
|
exp_set="$(printf '%s' "$want_set" | tr -d ' ' | tr ',' '\n' | sort | paste -sd, -)"
|
|
[[ "$got_set" == "$exp_set" ]] \
|
|
|| problems+=("caps: want exactly [$exp_set], got [$got_set]")
|
|
else
|
|
[[ "$got_caps" == *"$want"* ]] || problems+=("caps: want $want in [$got_caps]")
|
|
fi ;;
|
|
has)
|
|
IFS=',' read -ra want_keys <<< "$want"
|
|
for w in "${want_keys[@]}"; do
|
|
[[ ",$got_attached" == *",$w,"* ]] || problems+=("missing attachment: $w")
|
|
done ;;
|
|
hasnt)
|
|
IFS=',' read -ra bad_keys <<< "$want"
|
|
for b in "${bad_keys[@]}"; do
|
|
[[ ",$got_attached" == *",$b,"* ]] && problems+=("attached but must not be: $b")
|
|
done ;;
|
|
*)
|
|
problems+=("unknown assertion '$key' — typo, or a key this checker does not know") ;;
|
|
esac
|
|
done
|
|
|
|
if [[ ${#problems[@]} -eq 0 ]]; then
|
|
PASS=$((PASS + 1))
|
|
printf '\033[32m✓\033[0m %-6s %s\n' "$prof" "$q"
|
|
else
|
|
FAIL=$((FAIL + 1)); FAILED_LINES+=("$lineno")
|
|
printf '\033[31m✗\033[0m %-6s %s\n' "$prof" "$q"
|
|
for p in "${problems[@]}"; do echo " $p"; done
|
|
fi
|
|
|
|
[[ "$VERBOSE" == true ]] && printf '%s\n\n' "$report" | sed 's/^/ /'
|
|
done < "$FIXTURES"
|
|
|
|
echo
|
|
echo "─────────────────────────────────────────────"
|
|
printf 'passed %d failed %d' "$PASS" "$FAIL"
|
|
[[ $SKIP -gt 0 ]] && printf ' skipped %d' "$SKIP"
|
|
echo
|
|
if [[ $FAIL -gt 0 ]]; then
|
|
echo "failing fixture lines: ${FAILED_LINES[*]}"
|
|
exit 1
|
|
fi
|
|
exit 0
|