Pull partner token ledgers into a RAM cache so fleet totals are fleet-wide
This commit is contained in:
Gmer4Lfe
@@ -611,6 +611,14 @@ function vv_ai_token_prune(string $db): void {
|
||||
@rename($tmp, $db);
|
||||
}
|
||||
|
||||
// Partner ledgers pulled by AI/ai_token_sync.sh into tmpfs. Same trick conf_sync.sh uses for
|
||||
// partner confs: the reader treats a missing file as "unknown", never as zero.
|
||||
//
|
||||
// Mirrors AI_TOKEN_CACHE_DIR in load_config.sh. Hardcoded rather than read from conf because it
|
||||
// is a derived path constant on the shell side too — neither end reads it from a conf file, so
|
||||
// there is no single value to drift away from.
|
||||
const VV_AI_TOKEN_CACHE_DIR = '/tmp/.cache/vv/ai';
|
||||
|
||||
// Aggregates the file into today / last 7 days / all time, both overall and per host.
|
||||
//
|
||||
// Every detected host is present in 'hosts' whether or not it has rows, with 'seen' telling the
|
||||
@@ -629,50 +637,69 @@ function vv_ai_token_stats(): array {
|
||||
'first' => null, 'last' => null, 'best_tok_s' => null,
|
||||
];
|
||||
|
||||
$me = vv_detect_host();
|
||||
foreach (vv_known_hosts() as $id => $name) {
|
||||
$out['hosts'][$id] = ['name' => $name, 'seen' => false, 'self' => $id === vv_detect_host(),
|
||||
'today' => $blank, 'week' => $blank, 'all' => $blank];
|
||||
$out['hosts'][$id] = ['name' => $name, 'seen' => false, 'self' => $id === $me,
|
||||
'synced' => null, 'today' => $blank, 'week' => $blank, 'all' => $blank];
|
||||
}
|
||||
|
||||
$fh = @fopen($db, 'r');
|
||||
if (!$fh) return $out;
|
||||
// Ledgers to read: our own, plus whatever AI/ai_token_sync.sh has pulled from partners.
|
||||
//
|
||||
// Each entry carries the host slot it is allowed to contribute rows for. A partner file may
|
||||
// only add rows whose host column matches its filename — a ledger copied into the wrong slot,
|
||||
// or a partner that somehow cached ours, would otherwise be counted twice against a total
|
||||
// that still looked plausible. Our own file is trusted for any host, because it is the only
|
||||
// one written here and its host column is written by vv_detect_host().
|
||||
$ledgers = [[$db, null]];
|
||||
foreach ((array)@glob(VV_AI_TOKEN_CACHE_DIR . '/host*.tokens.db') as $partnerDb) {
|
||||
if (!preg_match('/(host\d+)\.tokens\.db$/', $partnerDb, $m)) continue;
|
||||
if ($m[1] === $me) continue; // our own ledger is already read above
|
||||
$ledgers[] = [$partnerDb, $m[1]];
|
||||
if (isset($out['hosts'][$m[1]])) $out['hosts'][$m[1]]['synced'] = @filemtime($partnerDb) ?: null;
|
||||
}
|
||||
|
||||
$add = function (array $b, int $p, int $c): array {
|
||||
$b['turns']++; $b['prompt'] += $p; $b['completion'] += $c; $b['total'] += $p + $c;
|
||||
return $b;
|
||||
};
|
||||
|
||||
while (($line = fgets($fh)) !== false) {
|
||||
$f = explode('|', rtrim($line, "\r\n"));
|
||||
if (count($f) < 8) continue; // partial write, or a format change
|
||||
[$date, , $host, $profile, $source, $p, $c, $tokS] = $f;
|
||||
if (!preg_match('/^\d{4}-\d{2}-\d{2}$/', $date)) continue;
|
||||
$p = (int)$p; $c = (int)$c;
|
||||
foreach ($ledgers as [$path, $onlyHost]) {
|
||||
$fh = @fopen($path, 'r');
|
||||
if (!$fh) continue;
|
||||
|
||||
if (!isset($out['hosts'][$host])) {
|
||||
// A host in the file that conf no longer lists — renamed, or a row copied in. Shown
|
||||
// rather than dropped, so the totals always reconcile with the per-host rows.
|
||||
$out['hosts'][$host] = ['name' => $host, 'seen' => false, 'self' => false,
|
||||
'today' => $blank, 'week' => $blank, 'all' => $blank];
|
||||
while (($line = fgets($fh)) !== false) {
|
||||
$f = explode('|', rtrim($line, "\r\n"));
|
||||
if (count($f) < 8) continue; // partial write, or a format change
|
||||
[$date, , $host, $profile, $source, $p, $c, $tokS] = $f;
|
||||
if (!preg_match('/^\d{4}-\d{2}-\d{2}$/', $date)) continue;
|
||||
if ($onlyHost !== null && $host !== $onlyHost) continue;
|
||||
$p = (int)$p; $c = (int)$c;
|
||||
|
||||
if (!isset($out['hosts'][$host])) {
|
||||
// A host in the file that conf no longer lists — renamed, or a row copied in.
|
||||
// Shown rather than dropped, so the totals reconcile with the per-host rows.
|
||||
$out['hosts'][$host] = ['name' => $host, 'seen' => false, 'self' => false,
|
||||
'synced' => null, 'today' => $blank, 'week' => $blank, 'all' => $blank];
|
||||
}
|
||||
$out['hosts'][$host]['seen'] = true;
|
||||
|
||||
foreach ([['all', true], ['week', $date >= $week], ['today', $date === $today]] as [$k, $hit]) {
|
||||
if (!$hit) continue;
|
||||
$out[$k] = $add($out[$k], $p, $c);
|
||||
$out['hosts'][$host][$k] = $add($out['hosts'][$host][$k], $p, $c);
|
||||
}
|
||||
|
||||
$out['profiles'][$profile] = ($out['profiles'][$profile] ?? 0) + $p + $c;
|
||||
$out['sources'][$source] = ($out['sources'][$source] ?? 0) + $p + $c;
|
||||
if ($tokS !== '' && (float)$tokS > (float)($out['best_tok_s'] ?? 0)) $out['best_tok_s'] = (float)$tokS;
|
||||
// Min/max rather than first-row/last-row. Appends are chronological in practice, but
|
||||
// a file hand-edited, merged or restored out of order would otherwise report a span
|
||||
// that reads backwards — and the row cap means the oldest row is not permanent.
|
||||
if ($out['first'] === null || $date < $out['first']) $out['first'] = $date;
|
||||
if ($out['last'] === null || $date > $out['last']) $out['last'] = $date;
|
||||
}
|
||||
$out['hosts'][$host]['seen'] = true;
|
||||
|
||||
foreach ([['all', true], ['week', $date >= $week], ['today', $date === $today]] as [$k, $hit]) {
|
||||
if (!$hit) continue;
|
||||
$out[$k] = $add($out[$k], $p, $c);
|
||||
$out['hosts'][$host][$k] = $add($out['hosts'][$host][$k], $p, $c);
|
||||
}
|
||||
|
||||
$out['profiles'][$profile] = ($out['profiles'][$profile] ?? 0) + $p + $c;
|
||||
$out['sources'][$source] = ($out['sources'][$source] ?? 0) + $p + $c;
|
||||
if ($tokS !== '' && (float)$tokS > (float)($out['best_tok_s'] ?? 0)) $out['best_tok_s'] = (float)$tokS;
|
||||
// Min/max rather than first-row/last-row. Appends are chronological in practice, but a
|
||||
// file that has been hand-edited, merged or restored out of order would otherwise report
|
||||
// a span that reads backwards — and the row cap means the oldest row is not permanent.
|
||||
if ($out['first'] === null || $date < $out['first']) $out['first'] = $date;
|
||||
if ($out['last'] === null || $date > $out['last']) $out['last'] = $date;
|
||||
fclose($fh);
|
||||
}
|
||||
fclose($fh);
|
||||
|
||||
arsort($out['profiles']);
|
||||
arsort($out['sources']);
|
||||
|
||||
@@ -473,9 +473,15 @@ if (is_dir('/var/log/varaverk')) {
|
||||
+ `<span class="vv-ai-hostrow-v">${num(tokData.all.total)}</span></div>`;
|
||||
Object.keys(hosts).forEach(id => {
|
||||
const x = hosts[id];
|
||||
// Three states, not two. "synced" is a partner whose ledger ai_token_sync.sh pulled;
|
||||
// "not collected here" is a partner we have never seen a ledger for. Collapsing the
|
||||
// second into a zero would report the partner as idle when the truth is we cannot see it.
|
||||
const tag = x.self ? ' · this host'
|
||||
: x.synced ? ' · synced ' + ago(x.synced)
|
||||
: '';
|
||||
hh += `<div class="vv-ai-hostrow${tokScope === id ? ' active' : ''}" data-scope="${esc(id)}">`
|
||||
+ `<span class="vv-ai-hostrow-n">${esc(x.name)}</span>`
|
||||
+ `<span class="vv-ai-hostrow-h">${esc(id)}${x.self ? ' · this host' : ''}</span>`
|
||||
+ `<span class="vv-ai-hostrow-h">${esc(id)}${tag}</span>`
|
||||
+ (x.seen ? `<span class="vv-ai-hostrow-v">${num(x.all.total)}</span>`
|
||||
: `<span class="vv-ai-hostrow-v none">not collected here</span>`)
|
||||
+ `</div>`;
|
||||
|
||||
Reference in New Issue
Block a user