<?php
// ═══════════════════════════════════════════════════════════════════════════════════════════════
// PURPOSE
//   The AI tab's library. Resolves AI configuration, reports the health of the retrieval
//   subsystem for the status banner, runs retrieval against the index, and owns the job-file
//   contract the chat worker and the polling endpoint share.
//
// OPERATIONAL MODEL
//   Retrieval is delegated, not reimplemented. AI/lib/search.js already embeds the query,
//   scans the index and applies intent routing; this file shells to that CLI with --json. A
//   second PHP implementation of vector search would be a second set of scoring bugs, and the
//   two would drift the first time the chunker changed.
//
//   Generation is not performed here. It takes 25-76 seconds and belongs in a detached worker;
//   this file only describes where that worker writes and what the states mean.
//
// DESIGN PRINCIPLES
//   Every reported statistic distinguishes "unknown" from "bad".
//       A banner that shows a healthy default when it cannot reach Ollama is worse than one
//       that shows nothing, because the operator stops checking. Each probe returns null when
//       it could not determine the answer.
//
//   Offload percentage is a first-class metric.
//       size_vram against size is the difference between 74 tok/s and 19 on this hardware, and
//       nothing else on the host surfaces it. It is computed here so the banner and any future
//       watchdog read the same number.
//
//   Staleness is measured against tracked files, not the filesystem.
//       The index only ever contains git-tracked content, so comparing to an untracked scratch
//       file would report permanent staleness. git ls-files is the same source the indexer uses.
//
//   Health checks compare configuration against reality, and carry their remedy.
//       "Retrieval failed" is not actionable; "HOST1_OLLAMA_MODEL names a tag that is not
//       installed" is. Every check names the setting to change, because the failures this
//       subsystem actually has are configuration drift — a model pulled, wired into conf, then
//       removed — rather than software faults.
//
//   Logs are read live, never indexed.
//       They churn constantly, would dominate a 3341-chunk corpus, and embedding similarity
//       retrieves log lines poorly compared with recency and a severity filter. Indexing them
//       would also mean re-embedding every few minutes for content that is stale immediately.
//
// OPERATIONAL SAFEGUARDS
//   Read-only. Nothing here writes to the index, the conf, or the job files — it reads state
//   and runs a retrieval query. The worker owns every write.
//
//   Every external call is time-boxed.
//       Retrieval and the Ollama probes carry explicit timeouts, because this library is
//       loaded by a page render and by a polling endpoint; a hung Ollama must degrade the
//       banner, not hang the tab.
//
//   Retrieval arguments are validated before they reach the shell.
//       kind is checked against the five real values and k is clamped, then every value is
//       escapeshellarg'd. An unknown kind would match zero rows in SQL and read as "the index
//       has no answer", which is the most misleading failure this subsystem can produce.
//
//   Tokens are validated as hex before they compose a path.
//       Job files live in a fixed directory and are named from a caller-supplied token; the
//       pattern is what keeps that token from escaping the directory.
//
//   A missing index is reported, never treated as empty.
//       Empty results and an absent index look identical to a caller that only counts rows,
//       and they need completely different responses from the user.
//
// EXPORTS
//   Config      vv_ai_enabled(), vv_ai_config()
//   Status      vv_ai_stats(), vv_ai_index_stats(), vv_ai_runtime_stats(), vv_ai_index_meta()
//   Models      vv_ai_models_available(), vv_ai_models_loaded()
//   Diagnosis   vv_ai_health(), vv_ai_recent_logs()
//   Memory      vv_ai_memory_path(), vv_ai_memory_read(), vv_ai_memory_write()
//   Retrieval   vv_ai_retrieve()
//   Jobs        vv_ai_job_dir(), vv_ai_job_path(), vv_ai_job_read()
//
// CONFIGURATION
//   master.conf   AI_ENABLED, AI_INDEX_DB, AI_SEARCH_K, AI_SEARCH_PER_FILE,
//                 AI_REQUEST_TIMEOUT, AI_CONNECT_TIMEOUT
//   host*.conf    HOST*_OLLAMA_URL, HOST*_OLLAMA_MODEL, HOST*_OLLAMA_EMBED_MODEL
// ═══════════════════════════════════════════════════════════════════════════════════════════════

require_once __DIR__ . '/config.php';

define('VV_AI_JOB_DIR', '/tmp/varaverk_ai_jobs');
const VV_AI_KINDS = ['header', 'readme', 'manual', 'template', 'doc'];

function vv_ai_config(): array {
    static $cfg = null;
    if ($cfg !== null) return $cfg;

    $vars = vv_conf_vars();
    $host = strtoupper(vv_detect_host());

    $cfg = [
        'enabled'     => strtolower(trim($vars['AI_ENABLED'] ?? 'false')) === 'true',
        'url'         => rtrim(trim($vars["{$host}_OLLAMA_URL"] ?? ''), '/'),
        'model'       => trim($vars["{$host}_OLLAMA_MODEL"] ?? ''),
        'embed_model' => trim($vars["{$host}_OLLAMA_EMBED_MODEL"] ?? 'nomic-embed-text'),
        'db'          => trim($vars['AI_INDEX_DB'] ?? '') ?: DATA_DIR . '/ai_index.db',
        'k'           => (int)($vars['AI_SEARCH_K'] ?? 8),
        'per_file'    => (int)($vars['AI_SEARCH_PER_FILE'] ?? 3),
        'timeout'     => (int)($vars['AI_REQUEST_TIMEOUT'] ?? 240),
        'connect'     => (int)($vars['AI_CONNECT_TIMEOUT'] ?? 5),
    ];
    // AI_INDEX_DB is written as "$DATA_DIR/ai_index.db" in conf; the shell expands it, PHP does
    // not. Left as a literal it would name a file that cannot exist.
    $cfg['db'] = str_replace(['$DATA_DIR', '${DATA_DIR}'], DATA_DIR, $cfg['db']);
    return $cfg;
}

function vv_ai_enabled(): bool {
    return vv_ai_config()['enabled'];
}

// Index size, coverage and staleness. Staleness compares the newest git-tracked file against
// the last build: the indexer only ever ingests tracked content, so anything else would report
// permanent drift.
function vv_ai_index_stats(): array {
    $cfg = vv_ai_config();
    $out = ['exists' => false, 'chunks' => 0, 'files' => 0, 'size' => 0,
            'built' => null, 'newest_source' => null, 'stale' => null, 'kinds' => []];

    if (!file_exists($cfg['db'])) return $out;
    $out['exists'] = true;
    $out['size']   = (int)@filesize($cfg['db']);

    $db = escapeshellarg($cfg['db']);
    $q  = fn(string $sql) => trim((string)@shell_exec('sqlite3 ' . $db . ' ' . escapeshellarg($sql) . ' 2>/dev/null'));

    $out['chunks'] = (int)$q('SELECT COUNT(*) FROM vv_chunks;');
    $out['files']  = (int)$q('SELECT COUNT(*) FROM vv_files;');
    $built         = (int)$q('SELECT MAX(indexed) FROM vv_files;');
    $out['built']  = $built ?: null;

    foreach (explode("\n", $q('SELECT kind, COUNT(*) FROM vv_chunks GROUP BY kind;')) as $line) {
        if (!$line) continue;
        [$k, $n] = array_pad(explode('|', $line, 2), 2, 0);
        if ($k !== '') $out['kinds'][$k] = (int)$n;
    }

    $newest = trim((string)@shell_exec(
        'cd ' . escapeshellarg(SCRIPTS_DIR) . ' && git ls-files -z 2>/dev/null'
        . ' | xargs -0 stat -c %Y 2>/dev/null | sort -rn | head -1'
    ));
    if ($newest !== '') {
        $out['newest_source'] = (int)$newest;
        if ($out['built'] !== null) $out['stale'] = (int)$newest > $out['built'];
    }
    return $out;
}

// Ollama reachability and — the number that matters on this hardware — whether the generation
// model is fully resident on the GPU. size_vram below size means layers are on the CPU, which
// costs roughly 4x throughput and is invisible everywhere else.
function vv_ai_runtime_stats(): array {
    $cfg = vv_ai_config();
    $out = ['reachable' => false, 'loaded' => null, 'offload_pct' => null,
            'context' => null, 'vram_used' => null, 'vram_total' => null, 'gpu' => null];

    if ($cfg['url'] === '') return $out;

    $ctx = stream_context_create(['http' => [
        'method' => 'GET', 'timeout' => max(2, $cfg['connect']), 'ignore_errors' => true,
    ]]);
    $raw = @file_get_contents($cfg['url'] . '/api/ps', false, $ctx);
    if ($raw === false) return $out;
    $out['reachable'] = true;

    $ps = json_decode($raw, true);
    foreach ($ps['models'] ?? [] as $m) {
        if (($m['name'] ?? '') !== $cfg['model']) continue;
        $size = (int)($m['size'] ?? 0);
        $vram = (int)($m['size_vram'] ?? 0);
        $out['loaded']      = true;
        $out['context']     = $m['context_length'] ?? null;
        $out['vram_used']   = $vram;
        $out['vram_total']  = $size;
        $out['offload_pct'] = $size > 0 ? (int)round($vram / $size * 100) : null;
        break;
    }
    if ($out['loaded'] === null) $out['loaded'] = false;

    $gpu = trim((string)@shell_exec(
        'nvidia-smi --query-gpu=name,memory.used,memory.total,utilization.gpu '
        . '--format=csv,noheader,nounits 2>/dev/null | tail -1'
    ));
    if ($gpu !== '') {
        $p = array_map('trim', explode(',', $gpu));
        if (count($p) >= 4) {
            $out['gpu'] = ['name' => $p[0], 'mem_used' => (int)$p[1],
                           'mem_total' => (int)$p[2], 'util' => (int)$p[3]];
        }
    }
    return $out;
}

// Models Ollama actually has on disk. The configured model naming a tag that does not exist is
// the single most likely misconfiguration here — it is what happens whenever a model is pulled,
// wired into conf, and later removed.
function vv_ai_models_available(): ?array {
    $cfg = vv_ai_config();
    if ($cfg['url'] === '') return null;
    $ctx = stream_context_create(['http' => ['timeout' => max(2, $cfg['connect']), 'ignore_errors' => true]]);
    $raw = @file_get_contents($cfg['url'] . '/api/tags', false, $ctx);
    if ($raw === false) return null;
    $d = json_decode($raw, true);
    if (!isset($d['models'])) return null;
    return array_map(fn($m) => [
        'name' => $m['name'] ?? '',
        'size' => (int)($m['size'] ?? 0),
    ], $d['models']);
}

// Currently resident models with their offload split.
function vv_ai_models_loaded(): ?array {
    $cfg = vv_ai_config();
    if ($cfg['url'] === '') return null;
    $ctx = stream_context_create(['http' => ['timeout' => max(2, $cfg['connect']), 'ignore_errors' => true]]);
    $raw = @file_get_contents($cfg['url'] . '/api/ps', false, $ctx);
    if ($raw === false) return null;
    $d = json_decode($raw, true);
    if (!isset($d['models'])) return null;
    $out = [];
    foreach ($d['models'] as $m) {
        $size = (int)($m['size'] ?? 0);
        $vram = (int)($m['size_vram'] ?? 0);
        $out[] = [
            'name'    => $m['name'] ?? '',
            'context' => $m['context_length'] ?? null,
            'size'    => $size,
            'vram'    => $vram,
            'offload' => $size > 0 ? (int)round($vram / $size * 100) : null,
        ];
    }
    return $out;
}

// vv_meta records the embed model and dimensionality the index was built with.
function vv_ai_index_meta(): array {
    $cfg = vv_ai_config();
    if (!file_exists($cfg['db'])) return [];
    // Columns are k/v, not key/value. Named wrongly this returns nothing and the embedder-match
    // check reports "unrecorded" — a silent pass on the one mismatch that returns confident
    // nonsense rather than an error.
    $raw = trim((string)@shell_exec(
        'sqlite3 ' . escapeshellarg($cfg['db']) . ' ' . escapeshellarg('SELECT k, v FROM vv_meta;') . ' 2>/dev/null'
    ));
    $out = [];
    foreach (explode("\n", $raw) as $line) {
        if ($line === '') continue;
        [$k, $v] = array_pad(explode('|', $line, 2), 2, '');
        if ($k !== '') $out[$k] = $v;
    }
    return $out;
}

// Config checked against reality. Each check is ok | warn | bad, with the remedy attached —
// the point is to name the setting that is wrong, not merely report that something failed.
function vv_ai_health(): array {
    $cfg    = vv_ai_config();
    $checks = [];
    $add = function (string $id, string $label, string $state, string $detail, string $fix = '')
        use (&$checks) {
        $checks[] = ['id' => $id, 'label' => $label, 'state' => $state,
                     'detail' => $detail, 'fix' => $fix];
    };

    $add('enabled', 'AI enabled', $cfg['enabled'] ? 'ok' : 'bad',
         $cfg['enabled'] ? 'AI_ENABLED=true' : 'AI_ENABLED is false',
         $cfg['enabled'] ? '' : 'Set AI_ENABLED=true in master.conf');

    if (!command_exists_node()) {
        $add('node', 'node runtime', 'bad', 'node not found on PATH',
             'Retrieval shells to AI/lib/cli.js and cannot run without it');
    }

    if ($cfg['url'] === '') {
        $add('url', 'Ollama URL', 'bad', 'not configured',
             'Set ' . strtoupper(vv_detect_host()) . '_OLLAMA_URL in the host conf');
        return $checks;
    }

    $avail = vv_ai_models_available();
    if ($avail === null) {
        $add('reach', 'Ollama reachable', 'bad', 'no response from ' . $cfg['url'],
             'Check the Ollama container is running and the URL is correct');
        return $checks;
    }
    $add('reach', 'Ollama reachable', 'ok', $cfg['url']);

    $names = array_column($avail, 'name');

    // The configured tag not existing is the failure this whole section is for.
    if ($cfg['model'] === '') {
        $add('gen', 'Generation model', 'bad', 'not configured',
             'Set ' . strtoupper(vv_detect_host()) . '_OLLAMA_MODEL in the host conf');
    } elseif (!vv_ai_model_installed($cfg['model'], $names)) {
        $add('gen', 'Generation model', 'bad', $cfg['model'] . ' is not installed',
             'Either `ollama pull` it, or point _OLLAMA_MODEL at one of: ' . implode(', ', array_slice($names, 0, 4)));
    } else {
        $add('gen', 'Generation model', 'ok', $cfg['model']);
    }

    if ($cfg['embed_model'] === '' || !vv_ai_model_installed($cfg['embed_model'], $names)) {
        $add('embed', 'Embedding model', 'bad',
             ($cfg['embed_model'] ?: 'not configured') . ' is not installed',
             'Retrieval cannot embed a query without it — `ollama pull ' . ($cfg['embed_model'] ?: 'nomic-embed-text') . '`');
    } else {
        $add('embed', 'Embedding model', 'ok', $cfg['embed_model']);
    }

    $ix   = vv_ai_index_stats();
    $meta = vv_ai_index_meta();

    if (!$ix['exists']) {
        $add('index', 'Index', 'bad', 'not built', 'Run AI/ai_index.sh');
    } else {
        $add('index', 'Index', 'ok', number_format($ix['chunks']) . ' chunks from ' . $ix['files'] . ' files');

        // A vector built by one embedding model is meaningless to another. Changing the embed
        // model without reindexing does not error — it silently returns nonsense, scored
        // confidently, which is the hardest failure here to notice from the answers alone.
        $builtWith = $meta['embed_model'] ?? '';
        if ($builtWith !== '' && $cfg['embed_model'] !== ''
            && vv_ai_model_norm($builtWith) !== vv_ai_model_norm($cfg['embed_model'])) {
            $add('embedmatch', 'Index / embedder match', 'bad',
                 'index built with ' . $builtWith . ', conf says ' . $cfg['embed_model'],
                 'Vectors from different models are not comparable — rerun AI/ai_index.sh --force');
        } else {
            $add('embedmatch', 'Index / embedder match', 'ok', $builtWith ?: 'unrecorded');
        }

        if ($ix['stale'] === true) {
            $add('fresh', 'Index freshness', 'warn', 'tracked files are newer than the index',
                 'Answers may cite code that has changed — rerun AI/ai_index.sh');
        } else {
            $add('fresh', 'Index freshness', 'ok', 'current');
        }
    }

    $loaded = vv_ai_models_loaded();
    if ($loaded !== null && $cfg['model'] !== '') {
        $hit = null;
        foreach ($loaded as $m) if ($m['name'] === $cfg['model']) { $hit = $m; break; }
        if ($hit === null) {
            $add('resident', 'Model resident', 'warn', 'not loaded — first question will load it');
        } elseif ($hit['offload'] !== null && $hit['offload'] < 100) {
            $add('resident', 'GPU offload', 'bad', $hit['offload'] . '% on GPU — layers on CPU',
                 'Roughly 4x slower on this card. Lower num_ctx or use a smaller quant');
        } else {
            $add('resident', 'GPU offload', 'ok', '100% on GPU at ' . ($hit['context'] ?? '?') . ' ctx');
        }
    }

    return $checks;
}

// Ollama reports tags fully qualified — nomic-embed-text:latest — while conf commonly carries
// the bare name, and both are valid references. Comparing literally reports an installed model
// as missing, which is exactly the false alarm that makes a health panel worth ignoring.
function vv_ai_model_norm(string $name): string {
    $name = trim($name);
    if ($name === '') return '';
    // A colon in the final path segment is a tag; anything else is a registry path.
    $last = substr($name, strrpos($name, '/') === false ? 0 : strrpos($name, '/') + 1);
    return str_contains($last, ':') ? $name : $name . ':latest';
}

function vv_ai_model_installed(string $name, array $available): bool {
    $n = vv_ai_model_norm($name);
    foreach ($available as $m) {
        if (vv_ai_model_norm($m) === $n) return true;
    }
    return false;
}

function command_exists_node(): bool {
    static $has = null;
    if ($has !== null) return $has;
    return $has = trim((string)@shell_exec('command -v node 2>/dev/null')) !== '';
}

// Recent warnings and errors across the orchestrator logs. Read live rather than indexed:
// logs churn constantly, would dominate a 3341-chunk index, and embedding similarity retrieves
// them poorly compared with recency plus a severity filter.
function vv_ai_recent_logs(int $max = 40): array {
    $dir = '/var/log/varaverk';
    if (!is_dir($dir)) return [];

    $cmd = 'grep -rhE "\[(ERROR|WARN|CRITICAL|FAILED)\]|✗" ' . escapeshellarg($dir)
         . ' --include="*.log" 2>/dev/null | tail -' . max(1, min($max, 200));
    $raw = (string)@shell_exec($cmd);

    $out = [];
    foreach (explode("\n", $raw) as $line) {
        $line = trim(preg_replace('/\033\[[0-9;]*[mK]/', '', $line));
        if ($line === '') continue;
        $out[] = mb_substr($line, 0, 300);
    }
    return $out;
}

function vv_ai_stats(): array {
    $cfg = vv_ai_config();
    return [
        'enabled'     => $cfg['enabled'],
        'model'       => $cfg['model'],
        'embed_model' => $cfg['embed_model'],
        'url'         => $cfg['url'],
        'k'           => $cfg['k'],
        'index'       => vv_ai_index_stats(),
        'runtime'     => vv_ai_runtime_stats(),
        'loaded'      => vv_ai_models_loaded(),
        'health'      => vv_ai_health(),
        'ts'          => time(),
    ];
}

// Retrieval via AI/lib/cli.js. Returns ['ok'=>bool,'results'=>[],'intents'=>[],'error'=>?string].
function vv_ai_retrieve(string $query, string $kind = '', string $section = '', ?int $k = null): array {
    $cfg = vv_ai_config();

    if ($query === '')                 return ['ok' => false, 'error' => 'query is required', 'results' => []];
    if ($cfg['url'] === '')            return ['ok' => false, 'error' => 'Ollama URL is not configured', 'results' => []];
    if (!file_exists($cfg['db']))      return ['ok' => false, 'error' => 'No index — run AI/ai_index.sh', 'results' => []];
    if ($kind !== '' && !in_array($kind, VV_AI_KINDS, true)) {
        return ['ok' => false, 'error' => 'Unknown kind: ' . $kind, 'results' => []];
    }

    $cli = SCRIPTS_DIR . '/AI/lib/cli.js';
    if (!file_exists($cli)) return ['ok' => false, 'error' => 'AI/lib/cli.js not found', 'results' => []];

    $k = max(1, min($k ?? $cfg['k'], 25));

    $cmd = 'timeout ' . max(10, $cfg['connect'] * 6) . ' node --no-warnings '
         . escapeshellarg($cli) . ' search'
         . ' --db=' . escapeshellarg($cfg['db'])
         . ' --url=' . escapeshellarg($cfg['url'])
         . ' --model=' . escapeshellarg($cfg['embed_model'])
         . ' --query=' . escapeshellarg($query)
         . ' --k=' . $k
         . ' --per-file=' . max(1, $cfg['per_file'])
         . ($kind !== ''    ? ' --kind='    . escapeshellarg($kind)    : '')
         . ($section !== '' ? ' --section=' . escapeshellarg($section) : '')
         . ' --json 2>/dev/null';

    $raw = trim((string)@shell_exec($cmd));
    if ($raw === '') return ['ok' => false, 'error' => 'Retrieval produced no output', 'results' => []];

    $d = json_decode($raw, true);
    if (!is_array($d) || !isset($d['results'])) {
        return ['ok' => false, 'error' => 'Retrieval returned unparseable output', 'results' => []];
    }
    return ['ok' => true, 'results' => $d['results'], 'intents' => $d['intents'] ?? [],
            'scanned' => $d['scanned'] ?? 0];
}

// ── Question shape ───────────────────────────────────────────────────────────────────────────
//
// "Varaverk" is a coined word with no spell-check and an awkward keyboard shape, so it arrives
// misspelled often — varavrk, veraverk, varavek, varverk. Edit distance catches those without a
// hand-written pattern that would need extending every time a new typo appears.
//
// This is search, not identity. The rule against fuzzy hostname matching is about deciding which
// machine you are talking to, where a near-miss must never resolve; here a near-miss only widens
// a document search, and the cost of being wrong is a slightly odd set of passages.

function vv_ai_mentions_varaverk(string $q): bool {
    foreach (preg_split('/[^a-z0-9]+/i', mb_strtolower($q), -1, PREG_SPLIT_NO_EMPTY) as $w) {
        if (strlen($w) < 5 || strlen($w) > 12) continue;   // bound the comparison window
        if ($w === 'varaverk' || levenshtein($w, 'varaverk') <= 2) return true;
    }
    return false;
}

// "What is Varaverk", "explain varavrk", "whats this thing do". Intent routing boosts PURPOSE
// for these, which surfaces every script's one-line purpose and buries the top-level prose that
// actually defines the system — so the model answers that the context does not define it, while
// README.md sits in the index unread.
function vv_ai_is_definitional(string $q): bool {
    if (!vv_ai_mentions_varaverk($q)) return false;
    return (bool)preg_match(
        '/\b(what\'?s?|whats|define|definition|describe|explain|overview|tell me about|'
        . 'purpose of|point of|elevator|in a nutshell)\b/i', $q);
}

// ── Memory ───────────────────────────────────────────────────────────────────────────────────
//
// A small operator-maintained file handed to the model at the start of every conversation.
// Injected, never indexed: it changes constantly, and vector similarity is the wrong retrieval
// mechanism for "things I was told to remember". Living under DATA_DIR keeps it out of the
// repository and therefore out of the index by construction.

function vv_ai_memory_path(): string {
    $cfg = vv_ai_config();
    $vars = vv_conf_vars();
    $p = trim($vars['AI_MEMORY_FILE'] ?? '');
    $p = str_replace(['$DATA_DIR', '${DATA_DIR}'], DATA_DIR, $p);
    return $p !== '' ? $p : DATA_DIR . '/ai_memory.md';
}

function vv_ai_memory_max(): int {
    $vars = vv_conf_vars();
    $n = (int)($vars['AI_MEMORY_MAX_CHARS'] ?? 4000);
    return max(200, min($n, 20000));
}

// Returns ['text'=>string,'chars'=>int,'truncated'=>bool,'exists'=>bool].
// Truncates rather than refusing: an over-long memory file should cost its own tail, not the
// whole conversation, and the notice tells the model its knowledge is incomplete rather than
// letting it assume it saw everything.
function vv_ai_memory_read(): array {
    $p   = vv_ai_memory_path();
    $max = vv_ai_memory_max();

    if (!file_exists($p)) return ['text' => '', 'chars' => 0, 'truncated' => false, 'exists' => false];

    $raw = (string)@file_get_contents($p);
    $len = mb_strlen($raw);
    if ($len <= $max) {
        return ['text' => $raw, 'chars' => $len, 'truncated' => false, 'exists' => true];
    }
    return [
        'text'      => mb_substr($raw, 0, $max) . "\n\n[memory truncated at {$max} characters]",
        'chars'     => $len,
        'truncated' => true,
        'exists'    => true,
    ];
}

// Atomic, and refuses to exceed the cap. The cap is a context budget shared with retrieval and
// reasoning on every turn, so it is enforced on write rather than silently trimmed on read.
function vv_ai_memory_write(string $text): array {
    $p   = vv_ai_memory_path();
    $max = vv_ai_memory_max();
    $len = mb_strlen($text);

    if ($len > $max) {
        return ['ok' => false, 'error' => "Memory is {$len} characters; the limit is {$max}. "
              . "It is included in every prompt, so it competes with retrieval for context."];
    }

    $dir = dirname($p);
    if (!is_dir($dir) && !@mkdir($dir, 0755, true)) {
        return ['ok' => false, 'error' => 'Cannot create ' . $dir];
    }

    $tmp = $p . '.vv.tmp';
    if (@file_put_contents($tmp, $text) === false) {
        return ['ok' => false, 'error' => 'Cannot write ' . $tmp];
    }
    if (!@rename($tmp, $p)) {
        @unlink($tmp);
        return ['ok' => false, 'error' => 'Cannot install ' . $p];
    }
    return ['ok' => true, 'chars' => $len];
}

// ── Token accounting ─────────────────────────────────────────────────────────────────────────
// One appended row per completed turn: date|time|host|profile|source|prompt|completion|tok_s.
// Pipe-delimited under data/ to match arr_cleanup_stats.db and bandwidth_history.db rather than
// inventing a format for one feature.
//
// Recording is best-effort and never raises. A turn that answered correctly must not be reported
// as failed because its accounting row could not be written — the number is the cheap part here
// and the answer is the expensive one.

function vv_ai_token_db(): string {
    $p = str_replace(['$DATA_DIR', '${DATA_DIR}'], DATA_DIR,
                     trim(vv_conf_vars()['AI_TOKEN_DB'] ?? ''));
    return $p !== '' ? $p : DATA_DIR . '/ai_token_history.db';
}

function vv_ai_token_retain(): int {
    $n = (int)(vv_conf_vars()['AI_TOKEN_RETAIN_ROWS'] ?? 20000);
    return max(500, min($n, 500000));
}

// LOCK_EX because two turns can finish together — the composer allows a second question while
// the first is still generating, and each runs in its own detached worker.
function vv_ai_token_record(string $profile, string $source, int $prompt, int $completion, ?float $tokS): void {
    if ($prompt <= 0 && $completion <= 0) return;   // nothing generated; not a turn worth a row

    $db  = vv_ai_token_db();
    $dir = dirname($db);
    if (!is_dir($dir) && !@mkdir($dir, 0755, true)) return;

    $row = implode('|', [
        date('Y-m-d'), date('H:i:s'),
        vv_detect_host(),
        preg_replace('/[^a-z0-9_-]/i', '', $profile) ?: 'unknown',
        preg_replace('/[^a-z0-9_-]/i', '', $source)  ?: 'unknown',
        max(0, $prompt), max(0, $completion),
        $tokS === null ? '' : number_format($tokS, 1, '.', ''),
    ]) . "\n";

    @file_put_contents($db, $row, FILE_APPEND | LOCK_EX);
    vv_ai_token_prune($db);
}

// Pruning is gated on filesize rather than done every turn: filesize() is a stat, whereas
// reading and rewriting the whole file is not something a chat turn should pay for per message.
// The threshold is derived from the row cap so raising the cap does not silently disable it.
function vv_ai_token_prune(string $db): void {
    $keep = vv_ai_token_retain();
    if ((int)@filesize($db) < $keep * 60) return;   // ~60 bytes a row, generously

    $rows = @file($db, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
    if (!is_array($rows) || count($rows) <= $keep) return;

    $tmp = $db . '.vv.tmp';
    if (@file_put_contents($tmp, implode("\n", array_slice($rows, -$keep)) . "\n") === false) return;
    @rename($tmp, $db);
}

// Partner ledgers pulled by AI/ai_token_sync.sh into tmpfs. Same trick conf_sync.sh uses for
// partner confs: the reader treats a missing file as "unknown", never as zero.
//
// Mirrors AI_TOKEN_CACHE_DIR in load_config.sh. Hardcoded rather than read from conf because it
// is a derived path constant on the shell side too — neither end reads it from a conf file, so
// there is no single value to drift away from.
const VV_AI_TOKEN_CACHE_DIR = '/tmp/.cache/vv/ai';

// Aggregates the file into today / last 7 days / all time, both overall and per host.
//
// Every detected host is present in 'hosts' whether or not it has rows, with 'seen' telling the
// two apart. A host with no rows is not the same fact as a host with zero tokens, and the page
// must be able to say "nothing collected here" rather than draw a confident 0.
function vv_ai_token_stats(): array {
    $db    = vv_ai_token_db();
    $today = date('Y-m-d');
    $week  = date('Y-m-d', strtotime('-6 days'));   // 7 days inclusive of today

    $blank = ['turns' => 0, 'prompt' => 0, 'completion' => 0, 'total' => 0];
    $out   = [
        'exists' => file_exists($db),
        'today'  => $blank, 'week' => $blank, 'all' => $blank,
        'hosts'  => [], 'profiles' => [], 'sources' => [],
        'first'  => null, 'last' => null, 'best_tok_s' => null,
    ];

    $me = vv_detect_host();
    foreach (vv_known_hosts() as $id => $name) {
        $out['hosts'][$id] = ['name' => $name, 'seen' => false, 'self' => $id === $me,
                              'synced' => null, 'today' => $blank, 'week' => $blank, 'all' => $blank];
    }

    // Ledgers to read: our own, plus whatever AI/ai_token_sync.sh has pulled from partners.
    //
    // Each entry carries the host slot it is allowed to contribute rows for. A partner file may
    // only add rows whose host column matches its filename — a ledger copied into the wrong slot,
    // or a partner that somehow cached ours, would otherwise be counted twice against a total
    // that still looked plausible. Our own file is trusted for any host, because it is the only
    // one written here and its host column is written by vv_detect_host().
    $ledgers = [[$db, null]];
    foreach ((array)@glob(VV_AI_TOKEN_CACHE_DIR . '/host*.tokens.db') as $partnerDb) {
        if (!preg_match('/(host\d+)\.tokens\.db$/', $partnerDb, $m)) continue;
        if ($m[1] === $me) continue;                       // our own ledger is already read above
        $ledgers[] = [$partnerDb, $m[1]];
        if (isset($out['hosts'][$m[1]])) $out['hosts'][$m[1]]['synced'] = @filemtime($partnerDb) ?: null;
    }

    $add = function (array $b, int $p, int $c): array {
        $b['turns']++; $b['prompt'] += $p; $b['completion'] += $c; $b['total'] += $p + $c;
        return $b;
    };

    foreach ($ledgers as [$path, $onlyHost]) {
        $fh = @fopen($path, 'r');
        if (!$fh) continue;

        while (($line = fgets($fh)) !== false) {
            $f = explode('|', rtrim($line, "\r\n"));
            if (count($f) < 8) continue;                   // partial write, or a format change
            [$date, , $host, $profile, $source, $p, $c, $tokS] = $f;
            if (!preg_match('/^\d{4}-\d{2}-\d{2}$/', $date)) continue;
            if ($onlyHost !== null && $host !== $onlyHost) continue;
            $p = (int)$p; $c = (int)$c;

            if (!isset($out['hosts'][$host])) {
                // A host in the file that conf no longer lists — renamed, or a row copied in.
                // Shown rather than dropped, so the totals reconcile with the per-host rows.
                $out['hosts'][$host] = ['name' => $host, 'seen' => false, 'self' => false,
                                        'synced' => null, 'today' => $blank, 'week' => $blank, 'all' => $blank];
            }
            $out['hosts'][$host]['seen'] = true;

            foreach ([['all', true], ['week', $date >= $week], ['today', $date === $today]] as [$k, $hit]) {
                if (!$hit) continue;
                $out[$k]                 = $add($out[$k], $p, $c);
                $out['hosts'][$host][$k] = $add($out['hosts'][$host][$k], $p, $c);
            }

            $out['profiles'][$profile] = ($out['profiles'][$profile] ?? 0) + $p + $c;
            $out['sources'][$source]   = ($out['sources'][$source]   ?? 0) + $p + $c;
            if ($tokS !== '' && (float)$tokS > (float)($out['best_tok_s'] ?? 0)) $out['best_tok_s'] = (float)$tokS;
            // Min/max rather than first-row/last-row. Appends are chronological in practice, but
            // a file hand-edited, merged or restored out of order would otherwise report a span
            // that reads backwards — and the row cap means the oldest row is not permanent.
            if ($out['first'] === null || $date < $out['first']) $out['first'] = $date;
            if ($out['last']  === null || $date > $out['last'])  $out['last']  = $date;
        }
        fclose($fh);
    }

    arsort($out['profiles']);
    arsort($out['sources']);
    $out['days'] = $out['first'] ? max(1, (int)((strtotime($today) - strtotime($out['first'])) / 86400) + 1) : 0;
    return $out;
}

function vv_ai_job_dir(): string {
    if (!is_dir(VV_AI_JOB_DIR)) @mkdir(VV_AI_JOB_DIR, 0700, true);
    return VV_AI_JOB_DIR;
}

// Hex-only, fixed length. The token composes a path, so the pattern is what confines it to the
// job directory.
function vv_ai_job_path(string $token): ?string {
    if (!preg_match('/^[0-9a-f]{32}$/', $token)) return null;
    return VV_AI_JOB_DIR . '/' . $token . '.json';
}

function vv_ai_job_read(string $token): ?array {
    $p = vv_ai_job_path($token);
    if ($p === null || !file_exists($p)) return null;
    $d = json_decode(@file_get_contents($p) ?: '', true);
    return is_array($d) ? $d : null;
}