Record token usage per turn and show daily, weekly and all-time totals by host

AI/README-AI.md

@@ -225,3 +225,38 @@ No cron entry and no `DAILY_MAINTENANCE_SCRIPTS` line are needed; the daily pull
 
 An incremental run on an unchanged repo is ~70 ms, so a daily entry costs effectively nothing
 and a pull that changed twelve files costs a few seconds.
+
+---
+
+## ━━━ TOKEN ACCOUNTING ━━━
+
+Every completed `ask` appends one row to `AI_TOKEN_DB` (`data/ai_token_history.db`):
+
+```
+date|time|host|profile|source|prompt_tokens|completion_tokens|tok_s
+2026-08-04|22:03:51|host1|varaverk|cli|2041|318|61.4
+```
+
+Both paths write it — this CLI (`source=cli`) and the WebGUI worker (`source=webgui`) — so the
+totals are not quietly the tab's alone. `ai_query.sh` passes `--token-db` and `--token-host`;
+called by hand without them, `cli.js` simply skips the row rather than guessing a path, because
+this file never reads conf itself.
+
+Read it on the plugin's AI tab, which aggregates today / last 7 days / all time, per host. Or
+straight from the shell, since it is just a delimited file:
+
+```bash
+# tokens used today
+awk -F'|' -v d="$(date +%F)" '$1==d {p+=$6; c+=$7} END {print p+c}' data/ai_token_history.db
+```
+
+**The host column is where the turn ran, not where the file is read.** Each host keeps its own
+`data/` and nothing syncs it, so a host only ever sees its own rows — the partner shows as
+"not collected here" on the tab, never as zero. Carrying a partner's totals would mean extending
+the partnership payload fetch; the column exists so that stays a display change rather than a
+migration.
+
+Pruning is by row count (`AI_TOKEN_RETAIN_ROWS`, default 20000) and happens on write, but only
+once the file passes a size threshold — an ordinary turn costs a `stat()` and an append. The CLI
+deliberately does not prune: duplicating a read-modify-write of the whole file in a second
+language is how the two drift apart.

AI/ai_query.sh

@@ -198,8 +198,12 @@ if [[ "$SEARCH_ONLY" == true ]]; then
         "--k=${AI_SEARCH_K:-8}" "--per-file=${AI_SEARCH_PER_FILE:-3}"
 else
     [[ -z "$GEN_MODEL" ]] && { error "${MY_ID}_OLLAMA_MODEL is empty — needed for generation"; exit 1; }
+    # Token accounting. Passed in rather than re-read in node, so the conf stays the shell's job
+    # and cli.js keeps taking everything it needs as arguments. Omitting either flag simply
+    # skips the row — the CLI must still work when called by hand outside this wrapper.
     node --no-warnings "$CLI" ask "${_args[@]}" \
         "--model=${GEN_MODEL}" "--embed-model=${EMBED_MODEL}" \
         "--k=${AI_SEARCH_K:-8}" "--per-file=${AI_SEARCH_PER_FILE:-3}" \
-        "--timeout=$(( ${AI_REQUEST_TIMEOUT:-240} * 1000 ))"
+        "--timeout=$(( ${AI_REQUEST_TIMEOUT:-240} * 1000 ))" \
+        "--token-db=${AI_TOKEN_DB:-}" "--token-host=$(echo "$MY_ID" | tr '[:upper:]' '[:lower:]')"
 fi

AI/lib/cli.js

@@ -12,6 +12,7 @@
 // are.
 // ═══════════════════════════════════════════════════════════════════════════════════════════════
 
+const fs             = require('fs');
 const { buildIndex } = require('./index.js');
 const { search }     = require('./search.js');
 
@@ -24,6 +25,27 @@ function flag(name) {
     return process.argv.includes(`--${name}`);
 }
 
+// Token accounting. Writes the same row shape as the WebGUI worker into the same file — one
+// ledger for both paths, or the totals quietly come to mean "whatever the tab happened to do".
+// Skipped silently when the caller passes neither flag, because cli.js has to stay runnable by
+// hand. Best-effort: a failed append must never cost a caller an answer it already has.
+//
+// Trimming is deliberately not done here. The PHP side prunes on write, and duplicating a
+// read-modify-write of the whole file in a second language is how the two drift apart.
+function recordTokens(profile, prompt, completion, tokS) {
+    const db = arg('token-db', ''), host = arg('token-host', '');
+    if (!db || !host || (prompt <= 0 && completion <= 0)) return;
+    const d  = new Date();
+    const p2 = n => String(n).padStart(2, '0');
+    const row = [
+        `${d.getFullYear()}-${p2(d.getMonth() + 1)}-${p2(d.getDate())}`,
+        `${p2(d.getHours())}:${p2(d.getMinutes())}:${p2(d.getSeconds())}`,
+        host, profile, 'cli', prompt, completion,
+        tokS === null ? '' : tokS.toFixed(1),
+    ].join('|') + '\n';
+    try { fs.appendFileSync(db, row); } catch { /* accounting is not the answer */ }
+}
+
 function fail(msg, code = 1) {
     console.error(msg);
     process.exit(code);
@@ -165,6 +187,11 @@ ANSWER`;
     if (!res.ok) fail(`generation HTTP ${res.status}`, 2);
     const j = await res.json();
 
+    // 'varaverk' rather than a CLI-specific name: this path retrieves and cites, so it is the
+    // same kind of turn the tab's default profile runs, and the two should aggregate together.
+    recordTokens('varaverk', j.prompt_eval_count || 0, j.eval_count || 0,
+                 j.eval_duration > 0 ? (j.eval_count / (j.eval_duration / 1e9)) : null);
+
     if (flag('json')) {
         console.log(JSON.stringify({ answer: j.response, sources: r.results.map(x => ({ path: x.path, section: x.section, heading: x.heading, score: x.score })) }));
         return;