diff --git a/AI/README-AI.md b/AI/README-AI.md index 72c9ea8..6e4aec6 100644 --- a/AI/README-AI.md +++ b/AI/README-AI.md @@ -250,11 +250,23 @@ straight from the shell, since it is just a delimited file: awk -F'|' -v d="$(date +%F)" '$1==d {p+=$6; c+=$7} END {print p+c}' data/ai_token_history.db ``` -**The host column is where the turn ran, not where the file is read.** Each host keeps its own -`data/` and nothing syncs it, so a host only ever sees its own rows — the partner shows as -"not collected here" on the tab, never as zero. Carrying a partner's totals would mean extending -the partnership payload fetch; the column exists so that stays a display change rather than a -migration. +**The host column is where the turn ran, not where the file is read.** Each host writes only its +own rows. + +`ai_token_sync.sh` pulls each partner's ledger into `/tmp/.cache/vv/ai/.tokens.db` — the +same trick `conf_sync.sh` uses for partner confs, and it runs from +`INTERMEDIATE_MAINTENANCE_SCRIPTS` every four hours. The tab then reads every ledger it can see, +so a fleet total is a fleet total. + +Pull only, no push: nothing here is needed by anyone else, and a reader that fetches its own data +controls its own freshness instead of depending on the partner's cron. A partner file may only +contribute rows whose `host` column matches its filename — a ledger copied into the wrong slot +would otherwise be double-counted against a total that still looked plausible. + +The cache is tmpfs with **no save/restore pair**, unlike the conf cache. Stale counters are worse +than absent ones: absent renders as "not collected here", stale renders as fact. An unreachable +partner leaves its file alone and logs at info, because a partner being down for weeks is a +normal state, not an incident. Pruning is by row count (`AI_TOKEN_RETAIN_ROWS`, default 20000) and happens on write, but only once the file passes a size threshold — an ordinary turn costs a `stat()` and an append. The CLI diff --git a/AI/ai_token_sync.sh b/AI/ai_token_sync.sh new file mode 100755 index 0000000..236c956 --- /dev/null +++ b/AI/ai_token_sync.sh @@ -0,0 +1,191 @@ +#!/bin/bash +# ============================================================================================== +# ============================= AI Token Ledger Sync =========================================== +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Pulls each partner's AI token ledger into a RAM cache at /tmp/.cache/vv/ai/, so the AI tab +# can report usage for the whole fleet instead of only the host the browser happens to be on. +# +# Each host records its own turns to data/ai_token_history.db and nothing syncs that file, so +# without this a host can only ever see its own totals. The tab is careful to say "not collected +# here" rather than 0 for a partner it cannot see; this script is what turns that into a number. +# +# Same trick as conf_sync.sh, and deliberately so — resolve the partner over Tailscale, scp one +# small file into a tmpfs cache, let the reader treat a missing file as "unknown". +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# 1. Gates — PARTNERSHIP_ENABLED, AI_ENABLED, AI_TOKEN_SYNC_ENABLED +# 2. Per partner: +# a. Resolve their Tailscale IP +# b. Resolve their SCRIPTS_DIR from their varaverk.cfg (they may be in appdata mode) +# c. scp their data/ai_token_history.db → $AI_TOKEN_CACHE_DIR/.tokens.db +# +# Pull only, no push. conf_sync.sh pushes as well because a partner may be unable to reach us +# and still needs our credentials; nothing here is needed by anyone else, and a reader that +# fetches its own data controls its own freshness rather than depending on the partner's cron. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# An unreachable partner is not a failure. +# HOST2 is expected to be down for long stretches during onboarding. A warn every four +# hours would train the operator to ignore this script's output, and the AI diagnostic +# path treats every log WARN as actionable. Unresolvable partners are counted and +# reported once at info level; only a partner that resolves and then fails to transfer +# is treated as an error. +# +# The cache is never written directly. +# scp lands on a .part file that is renamed into place, so a transfer interrupted halfway +# cannot leave the reader parsing half a ledger. A truncated final row would be skipped by +# the field-count check on the PHP side, but a torn file should not reach it at all. +# +# Nothing is ever written back to the partner. +# This script only reads. A bug here cannot corrupt a partner's accounting. +# +# The cache is tmpfs and deliberately not preserved. +# Unlike the conf cache there is no save/restore pair. Stale counters are worse than +# absent ones: absent reads as "not collected here", stale reads as fact. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# AI_ENABLED Whole AI subsystem gate +# AI_TOKEN_SYNC_ENABLED This script's own toggle (default: true) +# PARTNERSHIP_ENABLED Checked via require_partnership() +# SSH_KEY Key used for all partner ssh/scp operations +# +# load_config.sh +# +# AI_TOKEN_CACHE_DIR tmpfs directory the tab reads partner ledgers from +# +# host*.conf +# +# HOST* — hostnames used to build the partner list via detect_hosts() +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# ai_token_sync.sh Pull every reachable partner's ledger +# ai_token_sync.sh --dry-run Report what would be pulled, transfer nothing +# ai_token_sync.sh --log Verbose output +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../load_config.sh" + +parse_args "$@" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +acquire_lock + +detect_hosts +require_partnership + +if [[ "${AI_ENABLED:-false}" != true ]]; then + log "AI_ENABLED=false — skipping" + exit 0 +fi + +if [[ "${AI_TOKEN_SYNC_ENABLED:-true}" == false ]]; then + log "AI_TOKEN_SYNC_ENABLED=false — skipping" + exit 0 +fi + +CACHE_DIR="$AI_TOKEN_CACHE_DIR" +SSH_TIMEOUT=10 + +# Mirrors conf_sync.sh — the remote may be in appdata storage mode, so its ledger is not +# necessarily under /boot. +_remote_scripts_dir() { + local ip="$1" + local cfg line sd + cfg=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \ + "root@${ip}" "cat /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null" 2>/dev/null) || true + while IFS= read -r line; do + [[ "$line" == SCRIPTS_DIR=* ]] || continue + sd="${line#SCRIPTS_DIR=}"; sd="${sd//\"/}"; sd="${sd//\'/}" + echo "$sd"; return + done <<< "$cfg" + echo "/boot/config/plugins/varaverk" +} + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +if [[ "$DRY_RUN" == false ]]; then + mkdir -p "$CACHE_DIR" && chmod 755 "$CACHE_DIR" +fi + +PULLED=0 +OFFLINE=0 +FAILED=0 + +for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do + partner_host="${!host_var}" + [[ -z "$partner_host" ]] && continue + [[ "${host_var,,}" == "${MY_ID,,}" ]] && continue + + partner_slot="${host_var,,}" + partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true) + + if [[ -z "$partner_ip" ]]; then + log "$partner_host — offline or unresolvable, leaving its ledger absent" + (( OFFLINE++ )) + continue + fi + + remote_sd=$(_remote_scripts_dir "$partner_ip") + remote_db="${remote_sd}/data/ai_token_history.db" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would pull $partner_host:$remote_db → $CACHE_DIR/${partner_slot}.tokens.db" + continue + fi + + # A partner with AI off has no ledger at all. That is not an error — it is the same + # "nothing collected" the tab already knows how to render. + if ! timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \ + "root@${partner_ip}" "[[ -f '$remote_db' ]]" 2>/dev/null; then + log "$partner_host — no ledger on that host yet" + rm -f "$CACHE_DIR/${partner_slot}.tokens.db" + (( OFFLINE++ )) + continue + fi + + if timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \ + "root@${partner_ip}:${remote_db}" \ + "$CACHE_DIR/${partner_slot}.tokens.db.part" 2>/dev/null \ + && mv -f "$CACHE_DIR/${partner_slot}.tokens.db.part" "$CACHE_DIR/${partner_slot}.tokens.db"; then + chmod 644 "$CACHE_DIR/${partner_slot}.tokens.db" 2>/dev/null + echo "Pulled ${partner_slot} ledger from $partner_host ✅" + (( PULLED++ )) + else + rm -f "$CACHE_DIR/${partner_slot}.tokens.db.part" + warn "Could not pull ${partner_slot} ledger from $partner_host" + (( FAILED++ )) + fi +done + +info "AI token sync complete — pulled $PULLED${OFFLINE:+, $OFFLINE unavailable}${FAILED:+, $FAILED failed}" + +# Only a partner that answered and then failed the transfer is worth an exit code. An absent +# partner is the normal state while HOST2 is being rebuilt. +[[ "$FAILED" -gt 0 ]] && exit 1 +exit 0 diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index c6bb061..9fce12c 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -380,6 +380,7 @@ # to 1 day). 30min was true "always fresh" but had no consumer that needed it that often. "Arrs_Stack/arr_cache_prefill.sh ARR_PREFILL_WAIT_MINUTES=1" # keep the shared arr tracked-data cache fresh "Arrs_Stack/arrs_failed_stalled_recovery.sh" # blocklist + re-search failed/stalled arr queue items + "AI/ai_token_sync.sh" # pull partner AI token ledgers into the tmpfs cache ) # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here. # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above). @@ -1661,6 +1662,13 @@ AI_TOKEN_DB="$DATA_DIR/ai_token_history.db" AI_TOKEN_RETAIN_ROWS=20000 # oldest rows dropped past this — years of ordinary use +# AI/ai_token_sync.sh pulls each partner's ledger into the tmpfs cache the tab reads, so the +# fleet total is a fleet total. Same trick conf_sync.sh uses for partner confs, minus the push: +# nothing here is needed by anyone else, so the reader fetches its own data and controls its +# own freshness. An unreachable partner is a quiet skip, not a warning — a partner is expected +# to be down for long stretches, and a four-hourly warn trains you to ignore the script. + AI_TOKEN_SYNC_ENABLED=true + # ━━━ AI Feature Toggles ━━━ # Tier 1 is narration — it cannot change a decision. Tier 2 adds context to a decision a script # already made. Tier 3 assists a human. Enable in that order, and give each one weeks. diff --git a/Plugin/unraid/include/ai.php b/Plugin/unraid/include/ai.php index a59cfff..f1d5124 100644 --- a/Plugin/unraid/include/ai.php +++ b/Plugin/unraid/include/ai.php @@ -611,6 +611,14 @@ function vv_ai_token_prune(string $db): void { @rename($tmp, $db); } +// Partner ledgers pulled by AI/ai_token_sync.sh into tmpfs. Same trick conf_sync.sh uses for +// partner confs: the reader treats a missing file as "unknown", never as zero. +// +// Mirrors AI_TOKEN_CACHE_DIR in load_config.sh. Hardcoded rather than read from conf because it +// is a derived path constant on the shell side too — neither end reads it from a conf file, so +// there is no single value to drift away from. +const VV_AI_TOKEN_CACHE_DIR = '/tmp/.cache/vv/ai'; + // Aggregates the file into today / last 7 days / all time, both overall and per host. // // Every detected host is present in 'hosts' whether or not it has rows, with 'seen' telling the @@ -629,50 +637,69 @@ function vv_ai_token_stats(): array { 'first' => null, 'last' => null, 'best_tok_s' => null, ]; + $me = vv_detect_host(); foreach (vv_known_hosts() as $id => $name) { - $out['hosts'][$id] = ['name' => $name, 'seen' => false, 'self' => $id === vv_detect_host(), - 'today' => $blank, 'week' => $blank, 'all' => $blank]; + $out['hosts'][$id] = ['name' => $name, 'seen' => false, 'self' => $id === $me, + 'synced' => null, 'today' => $blank, 'week' => $blank, 'all' => $blank]; } - $fh = @fopen($db, 'r'); - if (!$fh) return $out; + // Ledgers to read: our own, plus whatever AI/ai_token_sync.sh has pulled from partners. + // + // Each entry carries the host slot it is allowed to contribute rows for. A partner file may + // only add rows whose host column matches its filename — a ledger copied into the wrong slot, + // or a partner that somehow cached ours, would otherwise be counted twice against a total + // that still looked plausible. Our own file is trusted for any host, because it is the only + // one written here and its host column is written by vv_detect_host(). + $ledgers = [[$db, null]]; + foreach ((array)@glob(VV_AI_TOKEN_CACHE_DIR . '/host*.tokens.db') as $partnerDb) { + if (!preg_match('/(host\d+)\.tokens\.db$/', $partnerDb, $m)) continue; + if ($m[1] === $me) continue; // our own ledger is already read above + $ledgers[] = [$partnerDb, $m[1]]; + if (isset($out['hosts'][$m[1]])) $out['hosts'][$m[1]]['synced'] = @filemtime($partnerDb) ?: null; + } $add = function (array $b, int $p, int $c): array { $b['turns']++; $b['prompt'] += $p; $b['completion'] += $c; $b['total'] += $p + $c; return $b; }; - while (($line = fgets($fh)) !== false) { - $f = explode('|', rtrim($line, "\r\n")); - if (count($f) < 8) continue; // partial write, or a format change - [$date, , $host, $profile, $source, $p, $c, $tokS] = $f; - if (!preg_match('/^\d{4}-\d{2}-\d{2}$/', $date)) continue; - $p = (int)$p; $c = (int)$c; + foreach ($ledgers as [$path, $onlyHost]) { + $fh = @fopen($path, 'r'); + if (!$fh) continue; - if (!isset($out['hosts'][$host])) { - // A host in the file that conf no longer lists — renamed, or a row copied in. Shown - // rather than dropped, so the totals always reconcile with the per-host rows. - $out['hosts'][$host] = ['name' => $host, 'seen' => false, 'self' => false, - 'today' => $blank, 'week' => $blank, 'all' => $blank]; + while (($line = fgets($fh)) !== false) { + $f = explode('|', rtrim($line, "\r\n")); + if (count($f) < 8) continue; // partial write, or a format change + [$date, , $host, $profile, $source, $p, $c, $tokS] = $f; + if (!preg_match('/^\d{4}-\d{2}-\d{2}$/', $date)) continue; + if ($onlyHost !== null && $host !== $onlyHost) continue; + $p = (int)$p; $c = (int)$c; + + if (!isset($out['hosts'][$host])) { + // A host in the file that conf no longer lists — renamed, or a row copied in. + // Shown rather than dropped, so the totals reconcile with the per-host rows. + $out['hosts'][$host] = ['name' => $host, 'seen' => false, 'self' => false, + 'synced' => null, 'today' => $blank, 'week' => $blank, 'all' => $blank]; + } + $out['hosts'][$host]['seen'] = true; + + foreach ([['all', true], ['week', $date >= $week], ['today', $date === $today]] as [$k, $hit]) { + if (!$hit) continue; + $out[$k] = $add($out[$k], $p, $c); + $out['hosts'][$host][$k] = $add($out['hosts'][$host][$k], $p, $c); + } + + $out['profiles'][$profile] = ($out['profiles'][$profile] ?? 0) + $p + $c; + $out['sources'][$source] = ($out['sources'][$source] ?? 0) + $p + $c; + if ($tokS !== '' && (float)$tokS > (float)($out['best_tok_s'] ?? 0)) $out['best_tok_s'] = (float)$tokS; + // Min/max rather than first-row/last-row. Appends are chronological in practice, but + // a file hand-edited, merged or restored out of order would otherwise report a span + // that reads backwards — and the row cap means the oldest row is not permanent. + if ($out['first'] === null || $date < $out['first']) $out['first'] = $date; + if ($out['last'] === null || $date > $out['last']) $out['last'] = $date; } - $out['hosts'][$host]['seen'] = true; - - foreach ([['all', true], ['week', $date >= $week], ['today', $date === $today]] as [$k, $hit]) { - if (!$hit) continue; - $out[$k] = $add($out[$k], $p, $c); - $out['hosts'][$host][$k] = $add($out['hosts'][$host][$k], $p, $c); - } - - $out['profiles'][$profile] = ($out['profiles'][$profile] ?? 0) + $p + $c; - $out['sources'][$source] = ($out['sources'][$source] ?? 0) + $p + $c; - if ($tokS !== '' && (float)$tokS > (float)($out['best_tok_s'] ?? 0)) $out['best_tok_s'] = (float)$tokS; - // Min/max rather than first-row/last-row. Appends are chronological in practice, but a - // file that has been hand-edited, merged or restored out of order would otherwise report - // a span that reads backwards — and the row cap means the oldest row is not permanent. - if ($out['first'] === null || $date < $out['first']) $out['first'] = $date; - if ($out['last'] === null || $date > $out['last']) $out['last'] = $date; + fclose($fh); } - fclose($fh); arsort($out['profiles']); arsort($out['sources']); diff --git a/Plugin/unraid/pages/ai.php b/Plugin/unraid/pages/ai.php index 7f6f8e1..fa10466 100644 --- a/Plugin/unraid/pages/ai.php +++ b/Plugin/unraid/pages/ai.php @@ -473,9 +473,15 @@ if (is_dir('/var/log/varaverk')) { + `${num(tokData.all.total)}`; Object.keys(hosts).forEach(id => { const x = hosts[id]; + // Three states, not two. "synced" is a partner whose ledger ai_token_sync.sh pulled; + // "not collected here" is a partner we have never seen a ledger for. Collapsing the + // second into a zero would report the partner as idle when the truth is we cannot see it. + const tag = x.self ? ' · this host' + : x.synced ? ' · synced ' + ago(x.synced) + : ''; hh += `
` + `${esc(x.name)}` - + `${esc(id)}${x.self ? ' · this host' : ''}` + + `${esc(id)}${tag}` + (x.seen ? `${num(x.all.total)}` : `not collected here`) + `
`; diff --git a/load_config.sh b/load_config.sh index e8ef7c8..ae26cc1 100755 --- a/load_config.sh +++ b/load_config.sh @@ -198,7 +198,13 @@ VV_CACHE_DIR="/tmp/vv_cache" CONF_RAM_CACHE_DIR="/tmp/.cache/vv/d" # tmpfs — cleared every reboot, repopulated by conf_sync.sh ARR_CACHE_DIR="/tmp/arr_cache" # tmpfs — cleared every reboot, restored from DATA_DIR backup by arr_cache_age_seconds() - export CONF_DIR LOG_DIR VV_CACHE_DIR CONF_RAM_CACHE_DIR ARR_CACHE_DIR + # Partner AI ledgers pulled by AI/ai_token_sync.sh. A sibling of the conf cache rather than + # the same directory: that one holds credentials at chmod 700 and is snapshotted to /boot by + # conf_cache_save.sh. These are non-secret counters that are worthless when stale, so they + # must not be preserved across a reboot — losing them just means "not collected here" until + # the next sync, which is the honest answer anyway. + AI_TOKEN_CACHE_DIR="/tmp/.cache/vv/ai" # tmpfs — cleared every reboot, repopulated by ai_token_sync.sh + export CONF_DIR LOG_DIR VV_CACHE_DIR CONF_RAM_CACHE_DIR ARR_CACHE_DIR AI_TOKEN_CACHE_DIR # ━━━ Cleanup ━━━ unset _conf _host_confs_loaded _adapter LOAD_CONFIG_DIR \ No newline at end of file