diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 258a23b..dc3b34f 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -1707,6 +1707,11 @@ # ━━━ AI Master Switch ━━━ # Fail-closed: anything other than the literal "true" means off. +# Which node runs the model, the retrieval index and the bug store. Every other node borrows it +# over the mesh, so a box without a GPU still gets the assistant — it just does not get the AI tab. +# Defaults to host1 when unset or malformed: whoever builds the mesh is host1. + AI_OWNER_HOST="host1" + AI_ENABLED=false AI_CONNECT_TIMEOUT=5 # seconds — probe when resolving which node has Ollama AI_REQUEST_TIMEOUT=240 # seconds — must clear a cold model load diff --git a/Plugin/unraid/Tools/ai_chat_worker.php b/Plugin/unraid/Tools/ai_chat_worker.php index 8f53b9a..2da2f00 100644 --- a/Plugin/unraid/Tools/ai_chat_worker.php +++ b/Plugin/unraid/Tools/ai_chat_worker.php @@ -998,8 +998,13 @@ $ctx = stream_context_create(['http' => [ $fh = @fopen($cfg['url'] . '/api/chat', 'r', false, $ctx); if ($fh === false) { + // Unpin, so the next question resolves to another node with a model rather than retrying a + // machine that is off. The error still names the endpoint that failed — "it moved on" is only + // useful alongside "here is what did not answer". + vv_ai_model_failed($cfg['model_host'] ?? ''); jw($jobFile, ['status' => 'error', - 'error' => 'Ollama did not respond within ' . max(30, $cfg['timeout']) . 's at ' . $cfg['url'], + 'error' => 'Ollama did not respond within ' . max(30, $cfg['timeout']) . 's at ' . $cfg['url'] + . (!empty($cfg['borrowed']) ? ' (borrowed from ' . $cfg['model_host'] . ')' : ''), 'sources' => $sources]); exit(1); } diff --git a/Plugin/unraid/Varaverk.page b/Plugin/unraid/Varaverk.page index 67a6515..5b45373 100644 --- a/Plugin/unraid/Varaverk.page +++ b/Plugin/unraid/Varaverk.page @@ -221,7 +221,10 @@ $validTabs = ['monitor', 'scheduler', 'docker', 'watchdog', 'partnership', 'fall // include/ai.php only ever reads the *local* {HOST}_OLLAMA_URL — there is no Tailscale resolver // in the PHP layer the way there is in the shell. On any other host the tab could only render // and then fail its own health check. -$_vv_ai = vv_ai_ui_on(); +// The tab is the owner-only surface — it carries the bug reports, the index and the model +// configuration. Assistant docks elsewhere use vv_ai_ui_on(), which every node with a reachable +// model passes. +$_vv_ai = vv_ai_owner_ui_on(); if ($_vv_ai) $validTabs[] = 'ai'; if (!in_array($tab, $validTabs)) $tab = 'monitor'; diff --git a/Plugin/unraid/api/ai.php b/Plugin/unraid/api/ai.php index 39dd52e..ae400dc 100644 --- a/Plugin/unraid/api/ai.php +++ b/Plugin/unraid/api/ai.php @@ -136,7 +136,7 @@ if ($action !== 'poll') { // on any host but HOST1, but a hidden link is not access control and this endpoint is reachable // directly. Every action is refused rather than just the expensive ones — there is no such thing // as a read this host is entitled to, since the index and the model are not here. -if (!vv_is_ai_host()) { +if (!vv_ai_is_owner()) { http_response_code(404); echo json_encode(['ok' => false, 'error' => 'AI is not available on this host']); exit; diff --git a/Plugin/unraid/include/ai.php b/Plugin/unraid/include/ai.php index 96b8a74..10def53 100644 --- a/Plugin/unraid/include/ai.php +++ b/Plugin/unraid/include/ai.php @@ -214,11 +214,33 @@ function vv_ai_config(): array { if ($cfg !== null) return $cfg; $vars = vv_conf_vars(); - $host = strtoupper(vv_detect_host()); + // The node whose model this one uses — itself if it has one, the owner's otherwise. Was + // vv_detect_host(), which meant a node without a GPU had no AI rather than borrowing one. + $modelHost = vv_ai_model_host(); + $host = strtoupper($modelHost); + + // Borrowed models are reached by hostname, not by the owner's own loopback. HOST1_OLLAMA_URL + // is http://localhost:11434, which is true on host1 and points at nothing anywhere else, so a + // remote borrower substitutes the owner's Tailscale name — the mesh resolves those and there + // are no hardcoded IPs anywhere in this project. + // + // The owner keeps loopback for its own calls. Rewriting it for everyone would push host1's own + // traffic onto the tailnet interface and make local AI depend on Tailscale being up, which is + // a real dependency to add for no gain. + $rawUrl = rtrim(trim($vars["{$host}_OLLAMA_URL"] ?? ''), '/'); + if ($modelHost !== vv_detect_host() && $rawUrl !== '') { + $ownerName = trim((string)($vars[strtoupper($modelHost)] ?? '')); + if ($ownerName !== '') { + $rawUrl = preg_replace('#://(localhost|127\.0\.0\.1)([:/]|$)#i', + '://' . $ownerName . '$2', $rawUrl); + } + } $cfg = [ 'enabled' => strtolower(trim($vars['AI_ENABLED'] ?? 'false')) === 'true', - 'url' => rtrim(trim($vars["{$host}_OLLAMA_URL"] ?? ''), '/'), + 'url' => $rawUrl, + 'model_host' => $modelHost, + 'borrowed' => $modelHost !== vv_detect_host(), 'model' => trim($vars["{$host}_OLLAMA_MODEL"] ?? ''), 'embed_model' => trim($vars["{$host}_OLLAMA_EMBED_MODEL"] ?? 'nomic-embed-text'), 'db' => trim($vars['AI_INDEX_DB'] ?? '') ?: AI_DATA_DIR . '/ai_index.db', diff --git a/Plugin/unraid/include/ai_repair.php b/Plugin/unraid/include/ai_repair.php index a1780e7..f4a593f 100644 --- a/Plugin/unraid/include/ai_repair.php +++ b/Plugin/unraid/include/ai_repair.php @@ -1164,7 +1164,10 @@ function vv_ai_ask_model(string $prompt, int $timeout = 120): ?string { 'ignore_errors' => true, ]]); $raw = @file_get_contents(rtrim($cfg['url'], '/') . '/api/chat', false, $ctx); - if ($raw === false) return null; + // Unpin on failure so the next resolution steps to another node that has a model. A refusal + // or a bad reply is not a failure of the endpoint — only not reaching it is, which is why + // this sits on the transport result and not on the parse below. + if ($raw === false) { vv_ai_model_failed($cfg['model_host'] ?? ''); return null; } $d = json_decode($raw, true); return $d['message']['content'] ?? null; } diff --git a/Plugin/unraid/include/config.php b/Plugin/unraid/include/config.php index 58596bb..98d7cc6 100644 --- a/Plugin/unraid/include/config.php +++ b/Plugin/unraid/include/config.php @@ -413,8 +413,94 @@ function vv_is_owner(): bool { // either moves, conflating them would move the other by accident. // // Returns false for 'unknown', so a host that cannot identify itself never shows the tab. +// Which node runs the model, the index and the bug store. Declared, not assumed: it was +// `=== 'host1'` for as long as host1 was the only box with a GPU, which made a physical fact look +// like a rule. A mesh can put the card anywhere — a spare GPU on someone else's node, a rebuilt +// host3 — and the owner has to be able to move without editing PHP. +// +// Defaults to host1 because whoever builds the mesh is host1, and is the person who would be +// changing this if it were ever wrong. +function vv_ai_owner_host(): string { + $h = strtolower(trim((string)(vv_conf_vars()['AI_OWNER_HOST'] ?? ''))); + return preg_match('/^host\d+$/', $h) ? $h : 'host1'; +} + +function vv_ai_is_owner(): bool { + return vv_detect_host() === vv_ai_owner_host(); +} + +// Whose Ollama this node should talk to: its own if it has one, otherwise the owner's. +// +// "Scans local, then defaults to the mesh" — a node with a GPU uses it, and a node without one +// borrows rather than going without. The check is whether the conf declares a URL for this host, +// not whether Ollama answers: a reachability probe on every call would spend a network round trip +// to decide where to send a network round trip, and a box whose own model is down wants that said +// plainly rather than papered over by silently using someone else's. +// Every node that declares a model, in the order this one should try them: itself first, then the +// owner, then anyone else. Local before mesh because a GPU you already have costs nothing to +// reach; owner before the rest because that is where the index and the curated model live. +function vv_ai_model_candidates(): array { + $vars = vv_conf_vars(); + $me = vv_detect_host(); + $owner = vv_ai_owner_host(); + $order = array_unique(array_merge([$me, $owner], array_keys(vv_known_hosts()))); + + $out = []; + foreach ($order as $h) { + if (trim((string)($vars[strtoupper($h) . '_OLLAMA_URL'] ?? '')) !== '') $out[] = $h; + } + return $out; +} + +// Where the resolved choice is remembered. VV_CACHE_ROOT is tmpfs, which is the right lifetime: +// a reboot re-resolves, and nothing about which node answered belongs on flash. +function vv_ai_model_pin_path(): string { + return rtrim(VV_CACHE_DIR, '/') . '/ai_model_host.json'; +} + +// The node whose model this one uses. Resolved once, then pinned until something fails. +// +// Pinned rather than re-derived because resolution is only interesting when it changes: a mesh +// where every request re-decides which node to ask is a mesh that will eventually decide +// differently mid-conversation, and a chat whose second turn goes to another machine has no +// history there. The pin is what makes "borrow the owner's model" a stable answer. +// +// Cleared by vv_ai_model_failed(), never by a timer. A working endpoint does not need rechecking, +// and an expiry would reintroduce exactly the mid-conversation switch the pin exists to stop. +function vv_ai_model_host(): string { + $cands = vv_ai_model_candidates(); + if (!$cands) return vv_ai_owner_host(); + + $pin = @json_decode((string)@file_get_contents(vv_ai_model_pin_path()), true); + if (is_array($pin) && in_array($pin['host'] ?? '', $cands, true)) return $pin['host']; + + $chosen = $cands[0]; + @file_put_contents(vv_ai_model_pin_path(), + json_encode(['host' => $chosen, 'at' => time()]), LOCK_EX); + return $chosen; +} + +// Called when a model call fails. Drops the pin and, if there is somewhere else to go, records the +// failed node so the next resolution steps past it rather than pinning it again. +// +// Deliberately forgets the failure as soon as the alternatives run out: a single-model mesh whose +// only node is briefly down should keep pointing at it and report that it is down, not resolve to +// nothing and report that AI is unconfigured. Those are different problems and only one of them is +// the operator's to fix. +function vv_ai_model_failed(string $host = ''): void { + $host = $host !== '' ? $host : vv_ai_model_host(); + $cands = vv_ai_model_candidates(); + $rest = array_values(array_filter($cands, fn($h) => $h !== $host)); + + if (!$rest) { @unlink(vv_ai_model_pin_path()); return; } + @file_put_contents(vv_ai_model_pin_path(), + json_encode(['host' => $rest[0], 'at' => time(), 'after_failure' => $host]), LOCK_EX); +} + +// Retained as the owner test it always was, so nothing that meant "is this the AI node" changes +// meaning underneath. Callers that meant "may this node use AI at all" want vv_ai_ui_on(). function vv_is_ai_host(): bool { - return vv_detect_host() === 'host1'; + return vv_ai_is_owner(); } // Whether the UI may offer anything AI at all: the right host, with the master switch on. Every @@ -425,8 +511,25 @@ function vv_is_ai_host(): bool { // that file; the Scheduler loads only config.php, and a gate that silently answers false where // its definition is missing is worse than no gate. Reads AI_ENABLED directly for the same // reason. Fail-closed on anything but the literal "true", matching the conf's own contract. +// May this node show AI features — the assistant docks, the findings strips, the AI rows on the +// Monitor card. No longer "am I host1": a node without a GPU borrows the owner's model over the +// mesh, so every node in the mesh gets the assistant. What it does not get is the AI tab; see +// vv_ai_owner_ui_on(). +// +// Still fails closed. A node with no local URL and no owner URL resolves to nothing, and an +// assistant that cannot reach a model is worse than an absent one. function vv_ai_ui_on(): bool { - return vv_is_ai_host() + if (strtolower(trim(vv_conf_vars()['AI_ENABLED'] ?? 'false')) !== 'true') return false; + $host = strtoupper(vv_ai_model_host()); + return trim((string)(vv_conf_vars()[$host . '_OLLAMA_URL'] ?? '')) !== ''; +} + +// May this node show the AI tab. Owner only, and deliberately so: that page carries the bug +// reports, the retrieval index and the model configuration — the surface where a wrong answer is +// expensive and the vocabulary assumes you built the thing. Someone running two containers on a +// node they were handed should have the assistant, not the machinery behind it. +function vv_ai_owner_ui_on(): bool { + return vv_ai_is_owner() && strtolower(trim(vv_conf_vars()['AI_ENABLED'] ?? 'false')) === 'true'; }