FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stream the answer as it is written, and let the operator stop it

Browse Source
This commit is contained in:
Gmer4Lfe
2026-08-10 20:22:57 -04:00
parent 9c744b34f8
commit eb24e60b8c
3 changed files with 245 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Plugin/unraid/Tools/ai_chat_worker.php
+66 -12
View File
@@ -126,6 +126,11 @@ if ($explain) {
if ($jobFile === '' || $question === '') exit(1);
if (!preg_match('#/[0-9a-f]{32}\.json$#', $jobFile)) exit(1);
// Claim the job immediately. Retrieval can take a second or two and a profile that skips it
// goes straight to the model, so without this the first write could be a streaming flush —
// leaving a Stop pressed early with no pid to signal and nothing to show for the press.
jw($jobFile, ['status' => 'starting']);
}
// What got attached and why, recorded as it happens rather than reconstructed afterwards.
@@ -174,9 +179,21 @@ $can = function (string $cap) use (&$profile): bool { return vv_ai_profile_can($
// Explain mode has no job file and no tab waiting on one, so the state writes are dropped rather
// than special-cased at each of their call sites.
// Atomic by way of rename(2), which is what makes streaming safe. While generating, this runs
// several times a second; a plain file_put_contents would eventually be caught mid-write by a
// poll, and vv_ai_job_read() json_decodes a truncated file to null, which the endpoint reports as
// "pending" — the page would read a job halfway through generating as one that had not started.
// The temp name carries the pid so two writers can never collide on it.
function jw(string $f, array $d): void {
if ($f === '') return;
file_put_contents($f, json_encode($d));
// The pid rides on every write rather than just the first, so Stop always has something to
// signal no matter which state the job is caught in — and so it cannot go missing when a
// later write forgets to carry it forward, which is exactly how the notify stamp was lost
// in the findings store.
if (!isset($d['pid'])) $d['pid'] = getmypid();
$tmp = $f . '.' . getmypid() . '.tmp';
if (@file_put_contents($tmp, json_encode($d)) === false) return;
if (!@rename($tmp, $f)) @unlink($tmp);
}
// Same log the endpoint writes to, tagged so the two are tellable apart. Defined here because
@@ -827,10 +844,15 @@ if ($explain) {
exit(0);
}
// Streamed rather than awaited. At ~61 t/s a long answer is several seconds of a spinner, and the
// wait is the same either way — but seeing the first line lands lets the operator tell in about a
// second whether the question was understood, instead of finding out at the end. It also makes
// "generating" a state that genuinely exists: before this the worker never wrote it, so the page
// showed "starting…" for the entire run.
$payload = json_encode([
'model' => $cfg['model'],
'messages' => $messages,
'stream' => false,
'stream' => true,
'think' => $think === '1',
'options' => ['num_ctx' => 16384],
]);
@@ -844,23 +866,55 @@ $ctx = stream_context_create(['http' => [
'ignore_errors' => true,
]]);
$raw = @file_get_contents($cfg['url'] . '/api/chat', false, $ctx);
if ($raw === false) {
$fh = @fopen($cfg['url'] . '/api/chat', 'r', false, $ctx);
if ($fh === false) {
jw($jobFile, ['status' => 'error',
'error' => 'Ollama did not respond within ' . max(30, $cfg['timeout']) . 's at ' . $cfg['url'],
'sources' => $sources]);
exit(1);
}
$d = json_decode($raw, true);
if (!is_array($d) || !isset($d['message'])) {
jw($jobFile, ['status' => 'error', 'error' => 'Unparseable response from Ollama',
'sources' => $sources]);
exit(1);
}
// Ollama streams NDJSON — one JSON object per line, each carrying a delta. The final object has
// done=true and is the only one holding the eval counters, so it is kept as $d for the ledger
// below; losing it would silently stop token accounting.
$answer = '';
$thinking = '';
$d = [];
$answer = trim((string)($d['message']['content'] ?? ''));
$thinking = trim((string)($d['message']['thinking'] ?? ''));
// Throttle. The job file is on tmpfs so writes are cheap, but the page polls on its own interval
// and rewriting faster than it reads is pure waste.
$FLUSH_SEC = 0.12;
$lastFlush = 0.0;
while (($line = fgets($fh)) !== false) {
$line = trim($line);
if ($line === '') continue;
$o = json_decode($line, true);
// A non-JSON line means Ollama answered with an error body rather than a stream. Nothing to
// accumulate; the empty-answer check below reports it.
if (!is_array($o)) continue;
if (isset($o['message']['content'])) $answer .= (string)$o['message']['content'];
if (isset($o['message']['thinking'])) $thinking .= (string)$o['message']['thinking'];
if (!empty($o['done'])) { $d = $o; break; }
$now = microtime(true);
if ($now - $lastFlush >= $FLUSH_SEC) {
$lastFlush = $now;
// thinking_chars rather than the reasoning itself: with think on, nothing lands in
// content for ~15s, and a page that only watched partial would look hung.
jw($jobFile, ['status' => 'generating',
'partial' => $answer,
'thinking_chars' => strlen($thinking),
'sources' => $sources]);
}
}
fclose($fh);
$answer = trim($answer);
$thinking = trim($thinking);
if ($answer === '') {
jw($jobFile, ['status' => 'error',
Plugin/unraid/api/ai.php
+62 -1
View File
@@ -207,6 +207,62 @@ if ($action === 'memory_set') {
exit;
}
// ── stop ──────────────────────────────────────────────────────────────────────
// Cancels a generation in flight. Only ever signals ONE pid, verified to be the worker for this
// exact job — never a process group. Signalling a group is what took the WebGUI down on
// 2026-08-07, and no group kill is needed here: the worker is a single php process whose only
// child-like thing is an HTTP connection to Ollama, which dies with it.
//
// Whatever was already generated is kept. A turn stopped at 80% is usually stopped because the
// operator has seen enough, not because they want it discarded.
if ($action === 'stop') {
if (!$isPost) { http_response_code(405); echo json_encode(['ok' => false, 'error' => 'POST only']); exit; }
$token = trim($_POST['token'] ?? '');
if (vv_ai_job_path($token) === null) {
echo json_encode(['ok' => false, 'error' => 'Invalid token']); exit;
}
$job = vv_ai_job_read($token);
if ($job === null) { echo json_encode(['ok' => false, 'error' => 'No such job']); exit; }
$status = (string)($job['status'] ?? '');
if ($status === 'done' || $status === 'error' || $status === 'stopped') {
echo json_encode(['ok' => true, 'already' => true, 'status' => $status]); exit;
}
$pid = (int)($job['pid'] ?? 0);
// Below 2 is init or nonsense. A pid we cannot verify is a pid we do not signal.
$killed = false;
if ($pid >= 2) {
// Pid reuse is the reason for this: the recorded worker may have exited seconds ago and
// the number been handed to something else entirely. The cmdline must name both this
// worker and this job's own file before anything is signalled.
$cmdline = @file_get_contents("/proc/$pid/cmdline");
$cmdline = $cmdline === false ? '' : str_replace("\0", ' ', $cmdline);
if (strpos($cmdline, 'ai_chat_worker.php') !== false && strpos($cmdline, $token) !== false) {
$killed = @posix_kill($pid, SIGTERM);
// No escalation ladder. The worker holds no lock and writes the job file atomically,
// so there is no cleanup that a delay would protect — and a SIGKILL race could land
// between the temp write and the rename.
}
}
// The job file is rewritten either way. If the pid could not be verified the worker is
// already gone, and the page still needs a terminal state instead of polling to its ceiling.
$job['status'] = 'stopped';
$job['stopped'] = true;
$job['answer'] = trim((string)($job['partial'] ?? $job['answer'] ?? ''));
unset($job['partial']);
@file_put_contents(vv_ai_job_path($token), json_encode($job));
vv_ai_log(sprintf('stop token=%s pid=%d signalled=%s kept=%d chars',
substr($token, 0, 12), $pid, $killed ? 'yes' : 'no', strlen($job['answer'])));
echo json_encode(['ok' => true, 'signalled' => $killed, 'kept' => strlen($job['answer'])]);
exit;
}
// ── clear ─────────────────────────────────────────────────────────────────────
if ($action === 'clear') {
if (!$isPost) { http_response_code(405); echo json_encode(['ok' => false, 'error' => 'POST only']); exit; }
@@ -433,7 +489,12 @@ if ($action === 'ask') {
exit;
}
$cmd = 'nohup php ' . escapeshellarg($worker) . ' '
// setsid, not just nohup. nohup detaches from the terminal but leaves the child in the
// caller's process group — php-fpm's. That is the arrangement that took the WebGUI down on
// 2026-08-07 when a Stop signalled a group it did not own. Stop below signals one verified
// pid and never a group, so this is belt and braces, but it also means a php-fpm restart no
// longer takes a running generation with it.
$cmd = 'setsid nohup php ' . escapeshellarg($worker) . ' '
. escapeshellarg($jobFile) . ' '
. escapeshellarg($question) . ' '
. escapeshellarg(json_encode($clean)) . ' '
Plugin/unraid/include/ai_chat.php
+117 -10
View File
@@ -190,6 +190,14 @@ function vv_ai_chat_assets(): void {
.vv-ai-offer-done { font-size:10px; color:#4a4a4a; font-style:italic; margin-top:6px; }
.vv-ai-pending { font-size:12px; color:#5a5a5a; display:flex; align-items:center; gap:8px; }
/* The streaming body sits in the pending bubble and is replaced wholesale by the finished message,
so it has to match .vv-ai-body or the answer visibly reflows the moment it completes. The caret
marks text as still arriving — without it a stream that pauses mid-sentence reads as finished. */
.vv-ai-stream { margin-top:6px; }
/* Stop reads as the destructive-ish action it is, and the colour change is what tells the operator
the button's job swapped — the label alone is easy to miss mid-answer. */
.vv-ai-btn.vv-ai-stop { background:#5a2b2b; border-color:#7a3b3b; color:#f0d8d8; }
.vv-ai-stream::after { content:'▌'; margin-left:1px; opacity:.55; animation:vvAiPulse 1.1s infinite; }
.vv-ai-dot { width:6px; height:6px; border-radius:50%; background:#6fcf97; animation:vvAiPulse 1.1s infinite; }
@keyframes vvAiPulse { 0%,100%{opacity:.25;} 50%{opacity:1;} }
@@ -402,6 +410,10 @@ vv_ai_profiles_script();
const onResize = o.onResize || function () {};
const store = o.chats !== false;
const POLL_MS = 1200;
// Once tokens are actually arriving, 1200ms delivers them in visible lumps and the stream reads
// as stuttering rather than writing. The endpoint only reads one small file off tmpfs, so the
// extra requests are cheap — and this rate only applies while a generation is in flight.
const POLL_FAST_MS = 350;
const POLL_CEIL = 300000; // stop polling a worker that never wrote a terminal state
// An instance already holding this prefix is a leftover from a tab swap, still holding
@@ -444,7 +456,8 @@ vv_ai_profiles_script();
const n = el(`<div class="vv-ai-msg bot" id="${P}-pending"><div class="vv-ai-role">Varaverk</div>`
+ `<div class="vv-ai-pending"><span class="vv-ai-dot"></span>`
+ `<span id="${P}-phase">starting…</span>`
+ `<span id="${P}-elapsed" style="color:#333;font-family:monospace"></span></div></div>`);
+ `<span id="${P}-elapsed" style="color:#333;font-family:monospace"></span></div>`
+ `<div class="vv-ai-body vv-ai-stream" id="${P}-stream" hidden></div></div>`);
chatEl().appendChild(n); scroll();
// An elapsed counter distinguishes "working" from "wedged" at a glance. Without it a
// stalled turn and a slow one look identical, and the slow case here is legitimately ~40s.
@@ -458,6 +471,37 @@ vv_ai_profiles_script();
}
function phase(t) { const p = $('phase'); if (p) p.textContent = t; }
// A line in the transcript that is not a turn — something the page did, said where the operator
// is already looking rather than in a banner they have to notice. Lives in the closure rather
// than only on the instance because the poll loop needs it too, and a bare note() there would
// throw: the instance object is not in scope from inside its own factory.
function noteLine(text) {
chatEl().appendChild(el('<div class="vv-ai-switch">' + esc(text) + '</div>'));
scroll();
}
// Streaming is only worth having if it does not fight the reader. Answers routinely outrun the
// panel, and the moment someone scrolls up to re-read a line, an unconditional scroll() on
// every delta drags them back down eight times a second. Stick to the bottom only while they
// are already there.
const nearBottom = () => {
const c = chatEl();
return (c.scrollHeight - c.scrollTop - c.clientHeight) < 40;
};
// Rendered through the same fmt() as a finished answer, so a fence that is still being written
// formats as it arrives rather than snapping from plain text to a code block at the end. fmt
// escapes, so a half-written tag cannot break out of the bubble mid-stream.
function streamInto(text) {
const s = $('stream');
if (!s) return;
if (!text) { s.hidden = true; return; }
const stick = nearBottom();
s.hidden = false;
s.innerHTML = fmt(text);
if (stick) scroll();
}
// Paths ride in data attributes rather than an onclick. They come out of the index, and a
// path carrying a quote interpolated into an attribute is code execution, not a display bug.
function sourcesHtml(sources) {
@@ -566,7 +610,10 @@ vv_ai_profiles_script();
if (o.beforeSend && o.beforeSend(q)) { $('input').value = ''; return; }
busy = true;
$('send').disabled = true;
// Ask becomes Stop rather than greying out. The one stretch of the interaction that takes
// 25-75s is exactly when the operator most wants a control, and the composer row has no
// room for a fourth button that is dead 95% of the time.
setSendMode('stop');
addUser(q);
$('input').value = '';
addPending();
@@ -625,6 +672,8 @@ vv_ai_profiles_script();
finish(); return; }
if (!d.ok) { addError(d.error || 'Failed to start'); finish(); return; }
messages.push({ role: 'user', content: q });
// Held for Stop. Set before the first poll so a press in the first second has a target.
curToken = d.token;
poll(d.token, Date.now());
})
.catch(e => { addError('Request failed: ' + (e && e.message ? e.message : e)); finish(); });
@@ -642,10 +691,39 @@ vv_ai_profiles_script();
function finish() {
busy = false;
const b = $('send'); if (b) b.disabled = false;
curToken = null;
setSendMode('ask');
clearInterval(pendingTimer);
}
// The token of the turn in flight. Held so Stop knows what to cancel, and cleared by finish()
// so a Stop pressed against an already-terminal job is impossible rather than merely harmless.
let curToken = null;
function setSendMode(mode) {
const b = $('send');
if (!b) return;
const stopping = (mode === 'stop');
b.disabled = false;
b.textContent = stopping ? 'Stop' : 'Ask';
b.classList.toggle('vv-ai-stop', stopping);
}
function stopTurn() {
if (!curToken) return;
const t = curToken;
// Disabled immediately: the kill is quick but the poll that renders the result is up to
// POLL_FAST_MS away, and a second press in that window would signal a dead pid.
const b = $('send'); if (b) b.disabled = true;
phase('stopping…');
fetch(API, { method: 'POST', headers: POST_HEAD,
body: new URLSearchParams({ action: 'stop', token: t }) })
.then(r => r.json())
.catch(() => {});
// Nothing is rendered from the response. The poll already owns turning a terminal state into
// a message, and having two paths do it is how a turn ends up in the transcript twice.
}
function poll(token, started) {
if (Date.now() - started > POLL_CEIL) {
addError('Timed out waiting for a response.'); finish(); return;
@@ -653,6 +731,25 @@ vv_ai_profiles_script();
fetch(API + '?action=poll&token=' + encodeURIComponent(token)).then(r => r.json()).then(d => {
if (!d.ok) { addError(d.error || 'Poll failed'); finish(); return; }
const j = d.job || {};
// Stopped is terminal and keeps whatever had been written. A turn cancelled at 80% is
// usually cancelled because there was already enough on screen, so discarding it would
// punish the operator for the one control that exists to save them time.
if (j.status === 'stopped') {
if ((j.answer || '').trim()) {
addAnswer(j);
messages.push({ role: 'assistant', content: j.answer });
noteLine('Stopped — the part already written is kept.');
} else {
const p = $('pending'); if (p) p.remove();
noteLine('Stopped before anything was written.');
}
fetch(API, { method: 'POST', headers: POST_HEAD,
body: new URLSearchParams({ action: 'clear', token }) }).catch(() => {});
finish();
save();
return;
}
if (j.status === 'done') {
addAnswer(j);
messages.push({ role: 'assistant', content: j.answer });
@@ -667,10 +764,21 @@ vv_ai_profiles_script();
return;
}
if (j.status === 'error') { addError(j.error || 'Unknown error'); finish(); return; }
// Partial text arrives on the same envelope as the status, so the transcript fills in
// without a second channel. Empty while the model is still reasoning — thinking_chars is
// what moves then, and a page watching only partial would look wedged for ~15s.
if (j.status === 'generating') streamInto(j.partial || '');
phase(j.status === 'generating'
? 'generating… (' + ((j.sources||[]).length) + ' sources retrieved)'
? (j.partial
? 'writing… (' + ((j.sources||[]).length) + ' sources)'
: (j.thinking_chars
? 'reasoning… (' + j.thinking_chars.toLocaleString() + ' chars)'
: 'generating… (' + ((j.sources||[]).length) + ' sources retrieved)'))
: j.status === 'retrieving' ? 'searching the index…' : 'starting…');
setTimeout(() => poll(token, started), POLL_MS);
setTimeout(() => poll(token, started),
j.status === 'generating' ? POLL_FAST_MS : POLL_MS);
}).catch(e => { addError('Poll failed: ' + e); finish(); });
}
@@ -905,7 +1013,9 @@ vv_ai_profiles_script();
// Ctrl+Enter is not bound here. It lives with the rest of the shortcuts on the wrapper below,
// and a second binding on the input would send the same question twice — the second landing
// on the busy guard and reporting the first as stuck.
$('send').addEventListener('click', send);
// One button, two jobs — which one is decided by busy rather than by what the label happens to
// say, so a stale label can never send a question into a turn already running.
$('send').addEventListener('click', () => { busy ? stopTurn() : send(); });
const newBtn = $('new'); if (newBtn) newBtn.addEventListener('click', newChat);
// ── Expand / collapse ────────────────────────────────────────────────
@@ -1169,10 +1279,7 @@ vv_ai_profiles_script();
// A line in the transcript that is not a turn — something the page did, said where the
// operator is already looking rather than in a banner they have to notice.
note(text) {
chatEl().appendChild(el('<div class="vv-ai-switch">' + esc(text) + '</div>'));
scroll();
},
note: noteLine,
teardown() {
clearInterval(pendingTimer);
// Would otherwise fire a re-layout at a page whose instance no longer exists.