178 lines
8.1 KiB
Bash
Executable File
178 lines
8.1 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# =============================== AI Explain Check =============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# Runs every case in ai_explain_fixtures.txt through the worker's --explain mode and checks the
|
|
# routing it reports against what the fixture says it should be. Catches a guard that has been
|
|
# undone by a later guard, which is the failure this subsystem keeps producing.
|
|
#
|
|
# OPERATIONAL MODEL
|
|
# Never calls the model. --explain stops on the line where deterministic assembly ends, so a
|
|
# full pass costs about a second per case and returns the same answer every time. There is no
|
|
# Ollama dependency, no token spend, and no flaky wording to chase.
|
|
#
|
|
# Not scheduled and deliberately not in any orchestrator. This is a development check — it runs
|
|
# when the routing changes, not every night. Nothing on the running system depends on it.
|
|
#
|
|
# DESIGN PRINCIPLES
|
|
# Asserts routing, never wording.
|
|
# Which capabilities a profile holds and which evidence was attached are decided before the
|
|
# model is asked anything. Asserting on generated prose would fail for reasons that tell
|
|
# nobody anything and the check would be ignored within a fortnight.
|
|
#
|
|
# A malformed assertion fails loudly rather than passing quietly.
|
|
# An unrecognised key is an error, not a skip. A typo in an assertion that silently passes
|
|
# is worse than no assertion, because the line still reads as covered.
|
|
#
|
|
# OPERATIONAL SAFEGUARDS
|
|
# Read-only. Runs fixtures through --explain and compares; writes nothing and changes no state.
|
|
#
|
|
# Exits non-zero on any failure, so it can gate a commit.
|
|
#
|
|
# RUNTIME MODES
|
|
# ai_explain_check.sh check every fixture
|
|
# ai_explain_check.sh --verbose print the full explain report for each case
|
|
# ai_explain_check.sh <pattern> only cases whose question matches the pattern
|
|
#
|
|
# DEPENDS ON
|
|
# Plugin/unraid/Tools/ai_chat_worker.php --explain mode
|
|
# Plugin/unraid/Tools/ai_explain_fixtures.txt
|
|
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
set -uo pipefail
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
WORKER="$HERE/ai_chat_worker.php"
|
|
FIXTURES="$HERE/ai_explain_fixtures.txt"
|
|
|
|
VERBOSE=false
|
|
FILTER=""
|
|
for a in "$@"; do
|
|
case "$a" in
|
|
--verbose) VERBOSE=true ;;
|
|
*) FILTER="$a" ;;
|
|
esac
|
|
done
|
|
|
|
[[ -f "$WORKER" ]] || { echo "missing worker: $WORKER"; exit 2; }
|
|
[[ -f "$FIXTURES" ]] || { echo "missing fixtures: $FIXTURES"; exit 2; }
|
|
|
|
PASS=0; FAIL=0; SKIP=0
|
|
FAILED_LINES=()
|
|
|
|
trim() { local s="$1"; s="${s#"${s%%[![:space:]]*}"}"; s="${s%"${s##*[![:space:]]}"}"; printf '%s' "$s"; }
|
|
|
|
# One field out of the explain report. Everything it reads is a fixed label printed by --explain.
|
|
field() {
|
|
local report="$1" label="$2"
|
|
printf '%s' "$report" | grep -m1 -E "^ *$label " | sed -E "s/^ *$label +//" | sed -E 's/ +$//'
|
|
}
|
|
|
|
lineno=0
|
|
while IFS= read -r raw || [[ -n "$raw" ]]; do
|
|
lineno=$((lineno + 1))
|
|
line="$(trim "$raw")"
|
|
[[ -z "$line" || "$line" == \#* ]] && continue
|
|
|
|
IFS='|' read -r q prof scope kind expect <<< "$line"
|
|
q="$(trim "$q")"; prof="$(trim "$prof")"; scope="$(trim "$scope")"
|
|
kind="$(trim "$kind")"; expect="$(trim "$expect")"
|
|
[[ -z "$q" ]] && continue
|
|
|
|
if [[ -n "$FILTER" && "$q" != *"$FILTER"* ]]; then SKIP=$((SKIP + 1)); continue; fi
|
|
|
|
report="$(php "$WORKER" --explain "$q" "$prof" "$scope" "$kind" 2>&1)"
|
|
if [[ $? -ne 0 ]]; then
|
|
echo "✗ line $lineno: explain failed — $q"
|
|
printf '%s\n' "$report" | head -3 | sed 's/^/ /'
|
|
FAIL=$((FAIL + 1)); FAILED_LINES+=("$lineno"); continue
|
|
fi
|
|
|
|
# PROFILE prints "asked -> used (escalated)" on a handoff and just the profile otherwise.
|
|
got_profile="$(printf '%s' "$report" | grep -m1 '^PROFILE' | sed -E 's/^PROFILE +//')"
|
|
got_profile="${got_profile##*-> }"
|
|
got_profile="$(printf '%s' "$got_profile" | sed -E 's/ *\(.*\)$//')"
|
|
got_caps="$(printf '%s' "$report" | grep -m1 '^CAPS' | sed -E 's/^CAPS +//')"
|
|
got_target="$(field "$report" 'named target')"
|
|
got_run="$(field "$report" 'run outcome')"
|
|
got_diag="$(field "$report" 'diagnostic')"
|
|
got_attached="$(printf '%s' "$report" | sed -n '/^ATTACHED/,/^$/p' | tail -n +2 \
|
|
| awk 'NF {print $1}' | tr '\n' ',' )"
|
|
|
|
problems=()
|
|
for assert in $expect; do
|
|
key="${assert%%=*}"; want="${assert#*=}"
|
|
case "$key" in
|
|
profile)
|
|
[[ "$got_profile" == "$want" ]] || problems+=("profile: want $want, got $got_profile") ;;
|
|
target)
|
|
if [[ "$want" == "none" ]]; then
|
|
[[ "$got_target" == "(none resolved)" ]] || problems+=("target: want none, got $got_target")
|
|
else
|
|
[[ "$got_target" == "$want" ]] || problems+=("target: want $want, got $got_target")
|
|
fi ;;
|
|
run)
|
|
exp=$([[ "$want" == "yes" ]] && echo YES || echo no)
|
|
[[ "$got_run" == "$exp" ]] || problems+=("run outcome: want $exp, got $got_run") ;;
|
|
diag)
|
|
exp=$([[ "$want" == "yes" ]] && echo YES || echo no)
|
|
[[ "$got_diag" == "$exp" ]] || problems+=("diagnostic: want $exp, got $got_diag") ;;
|
|
caps)
|
|
# none holds nothing at all
|
|
# only:a,b holds exactly these and nothing else
|
|
# a holds at least this
|
|
#
|
|
# only: exists because General Chat's guarantee is not "it can search" but "search
|
|
# is the only thing it can do". A presence check would still pass on the day
|
|
# something else is granted there, which is the day the check was written for.
|
|
if [[ "$want" == "none" ]]; then
|
|
[[ "$got_caps" == "(none"* ]] || problems+=("caps: want none, got $got_caps")
|
|
elif [[ "$want" == only:* ]]; then
|
|
want_set="${want#only:}"
|
|
got_set="$(printf '%s' "$got_caps" | tr -d ' ' | tr ',' '\n' | sort | paste -sd, -)"
|
|
exp_set="$(printf '%s' "$want_set" | tr -d ' ' | tr ',' '\n' | sort | paste -sd, -)"
|
|
[[ "$got_set" == "$exp_set" ]] \
|
|
|| problems+=("caps: want exactly [$exp_set], got [$got_set]")
|
|
else
|
|
[[ "$got_caps" == *"$want"* ]] || problems+=("caps: want $want in [$got_caps]")
|
|
fi ;;
|
|
has)
|
|
IFS=',' read -ra want_keys <<< "$want"
|
|
for w in "${want_keys[@]}"; do
|
|
[[ ",$got_attached" == *",$w,"* ]] || problems+=("missing attachment: $w")
|
|
done ;;
|
|
hasnt)
|
|
IFS=',' read -ra bad_keys <<< "$want"
|
|
for b in "${bad_keys[@]}"; do
|
|
[[ ",$got_attached" == *",$b,"* ]] && problems+=("attached but must not be: $b")
|
|
done ;;
|
|
*)
|
|
problems+=("unknown assertion '$key' — typo, or a key this checker does not know") ;;
|
|
esac
|
|
done
|
|
|
|
if [[ ${#problems[@]} -eq 0 ]]; then
|
|
PASS=$((PASS + 1))
|
|
printf '\033[32m✓\033[0m %-6s %s\n' "$prof" "$q"
|
|
else
|
|
FAIL=$((FAIL + 1)); FAILED_LINES+=("$lineno")
|
|
printf '\033[31m✗\033[0m %-6s %s\n' "$prof" "$q"
|
|
for p in "${problems[@]}"; do echo " $p"; done
|
|
fi
|
|
|
|
[[ "$VERBOSE" == true ]] && printf '%s\n\n' "$report" | sed 's/^/ /'
|
|
done < "$FIXTURES"
|
|
|
|
echo
|
|
echo "─────────────────────────────────────────────"
|
|
printf 'passed %d failed %d' "$PASS" "$FAIL"
|
|
[[ $SKIP -gt 0 ]] && printf ' skipped %d' "$SKIP"
|
|
echo
|
|
if [[ $FAIL -gt 0 ]]; then
|
|
echo "failing fixture lines: ${FAILED_LINES[*]}"
|
|
exit 1
|
|
fi
|
|
exit 0
|