Gmer4Lfe
c0da7d935d
The pattern used # as its delimiter and excluded a literal # inside the character class, so PCRE ended the pattern early and the match never fired. It read as the filter doing nothing. The unit test missed it by asserting against a retyped copy of the regex rather than the one in the file, which is the only version that matters.
285 lines
14 KiB
PHP
285 lines
14 KiB
PHP
<?php
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
// PURPOSE
|
|
// Web search for the chat profile. Turns a question into a handful of titled snippets with
|
|
// their URLs, for the worker to put in front of the model as context it can cite.
|
|
//
|
|
// OPERATIONAL MODEL
|
|
// One provider at a time, named by AI_WEB_SEARCH_PROVIDER. Four are supported because the right
|
|
// answer depends on what the operator already runs: degoog and SearXNG are self-hosted and need
|
|
// no key and no third party, while brave and tavily need no container. They differ only in how
|
|
// the request is shaped and where the fields sit in the reply, so everything after parsing is
|
|
// one code path.
|
|
//
|
|
// degoog is the default and the one verified here — it was already running on this host, and it
|
|
// aggregates several engines and returns them merged. DuckDuckGo is deliberately absent: both
|
|
// its scraping endpoints answer a challenge page from this network, and its keyless API returns
|
|
// nothing at all for a real question.
|
|
//
|
|
// DESIGN PRINCIPLES
|
|
// Off by default, and not because it is dangerous.
|
|
// Searching sends the operator's question to something outside this house. That is a
|
|
// decision for the person whose question it is, so AI_WEB_SEARCH_ENABLED starts false and
|
|
// nothing turns it on.
|
|
//
|
|
// Chat only, and that is a capability, not a check in here.
|
|
// General chat holds no other capability precisely because it cannot write anything. Search
|
|
// is a read, so it is the one capability that fits there. The Varaverk assistant does not
|
|
// get it: its whole contract is that answers come from this installation's own documents,
|
|
// and a web result in that context is an answer that looks sourced and is not.
|
|
//
|
|
// Snippets, never pages.
|
|
// Only what the search API returns. Fetching the pages themselves would mean running an
|
|
// HTML parser over whatever the internet handed back, on a box that already has enough to
|
|
// do, for text the model has a 16k window to hold.
|
|
//
|
|
// OPERATIONAL SAFEGUARDS
|
|
// Bounded everywhere. A timeout on the call, a cap on the number of results, and a cap on the
|
|
// length of each snippet — the context this competes for is the same one retrieval needs.
|
|
//
|
|
// The query is the operator's question and nothing else. No conf values, no host names, no log
|
|
// contents are ever appended to it: this is the one path in the plugin that sends text off the
|
|
// machine, and what leaves has to be exactly what the operator typed.
|
|
//
|
|
// Redacted before it is logged, like every other question in this subsystem.
|
|
//
|
|
// DEPENDS ON
|
|
// include/ai.php vv_ai_redact(), vv_conf_vars()
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
require_once __DIR__ . '/ai.php';
|
|
|
|
const VV_AI_WEB_PROVIDERS = ['degoog', 'searxng', 'brave', 'tavily'];
|
|
const VV_AI_WEB_MAX_QUERY = 300; // characters sent to the provider
|
|
const VV_AI_WEB_MAX_SNIPPET = 400; // characters kept from each result
|
|
|
|
function vv_ai_web_enabled(): bool {
|
|
if (!vv_ai_enabled()) return false;
|
|
return strtolower(trim((string)(vv_conf_vars()['AI_WEB_SEARCH_ENABLED'] ?? 'false'))) === 'true';
|
|
}
|
|
|
|
function vv_ai_web_provider(): string {
|
|
$p = strtolower(trim((string)(vv_conf_vars()['AI_WEB_SEARCH_PROVIDER'] ?? 'degoog')));
|
|
return in_array($p, VV_AI_WEB_PROVIDERS, true) ? $p : 'degoog';
|
|
}
|
|
|
|
function vv_ai_web_results_max(): int {
|
|
return max(1, min(10, (int)(vv_conf_vars()['AI_WEB_SEARCH_RESULTS'] ?? 4)));
|
|
}
|
|
|
|
function vv_ai_web_timeout(): int {
|
|
return max(2, min(30, (int)(vv_conf_vars()['AI_WEB_SEARCH_TIMEOUT'] ?? 6)));
|
|
}
|
|
|
|
// Per-host, because a SearXNG URL is an address on this network and an API key is a credential —
|
|
// both belong in host*.conf by the same rule that puts every other one there.
|
|
function vv_ai_web_searx_url(): string {
|
|
return rtrim(trim((string)(vv_conf_vars()[strtoupper(vv_detect_host()) . '_SEARXNG_URL'] ?? '')), '/');
|
|
}
|
|
|
|
function vv_ai_web_degoog_url(): string {
|
|
return rtrim(trim((string)(vv_conf_vars()[strtoupper(vv_detect_host()) . '_DEGOOG_URL'] ?? '')), '/');
|
|
}
|
|
|
|
// Which providers need an address on this network rather than a key. Kept as a list because the
|
|
// readiness check and the error message both need the same answer and disagreeing about it is how
|
|
// a feature reports itself ready and then does nothing.
|
|
const VV_AI_WEB_SELF_HOSTED = ['degoog', 'searxng'];
|
|
|
|
function vv_ai_web_api_key(): string {
|
|
return trim((string)(vv_conf_vars()[strtoupper(vv_detect_host()) . '_WEB_SEARCH_API_KEY'] ?? ''));
|
|
}
|
|
|
|
// Is this provider actually set up? Answered separately from "is it switched on", because the
|
|
// two failures need different sentences: one is a setting, the other is a missing address or key.
|
|
function vv_ai_web_ready(): array {
|
|
if (!vv_ai_web_enabled()) {
|
|
return ['ok' => false, 'error' => 'AI_WEB_SEARCH_ENABLED is false'];
|
|
}
|
|
$p = vv_ai_web_provider();
|
|
$host = strtoupper(vv_detect_host());
|
|
|
|
if ($p === 'degoog' && vv_ai_web_degoog_url() === '') {
|
|
return ['ok' => false, 'error' => $host . '_DEGOOG_URL is empty'];
|
|
}
|
|
if ($p === 'searxng' && vv_ai_web_searx_url() === '') {
|
|
return ['ok' => false, 'error' => $host . '_SEARXNG_URL is empty'];
|
|
}
|
|
if (!in_array($p, VV_AI_WEB_SELF_HOSTED, true) && vv_ai_web_api_key() === '') {
|
|
return ['ok' => false, 'error' => $host . '_WEB_SEARCH_API_KEY is empty'];
|
|
}
|
|
return ['ok' => true, 'provider' => $p];
|
|
}
|
|
|
|
// One HTTP call, returning the decoded body or null. curl rather than file_get_contents so the
|
|
// timeout is real and a proxy-less environment does not hang the worker.
|
|
function vv_ai_web_http(string $url, array $headers = [], ?array $postJson = null): ?array {
|
|
$ch = curl_init($url);
|
|
if ($ch === false) return null;
|
|
curl_setopt_array($ch, [
|
|
CURLOPT_RETURNTRANSFER => true,
|
|
CURLOPT_TIMEOUT => vv_ai_web_timeout(),
|
|
CURLOPT_CONNECTTIMEOUT => min(5, vv_ai_web_timeout()),
|
|
CURLOPT_FOLLOWLOCATION => false,
|
|
CURLOPT_HTTPHEADER => array_merge(['Accept: application/json'], $headers),
|
|
CURLOPT_USERAGENT => 'Varaverk',
|
|
]);
|
|
if ($postJson !== null) {
|
|
curl_setopt($ch, CURLOPT_POST, true);
|
|
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($postJson));
|
|
curl_setopt($ch, CURLOPT_HTTPHEADER,
|
|
array_merge(['Accept: application/json', 'Content-Type: application/json'], $headers));
|
|
}
|
|
$body = curl_exec($ch);
|
|
$code = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
|
curl_close($ch);
|
|
if ($body === false || $code < 200 || $code >= 300) return null;
|
|
$d = json_decode((string)$body, true);
|
|
return is_array($d) ? $d : null;
|
|
}
|
|
|
|
// Everything below reduces to the same three fields, so the worker never learns which provider
|
|
// answered: title, url, snippet.
|
|
function vv_ai_web_normalise(array $rows): array {
|
|
$out = [];
|
|
foreach ($rows as $r) {
|
|
$url = trim((string)($r['url'] ?? ''));
|
|
// Only http(s). A provider returning anything else is either broken or hostile, and this
|
|
// string ends up in an href.
|
|
if (!preg_match('#^https?://#i', $url)) continue;
|
|
$title = trim((string)($r['title'] ?? ''));
|
|
$snip = trim(preg_replace('/\s+/u', ' ', (string)($r['snippet'] ?? '')));
|
|
if ($title === '' && $snip === '') continue;
|
|
$out[] = ['title' => mb_substr($title !== '' ? $title : $url, 0, 160),
|
|
'url' => $url,
|
|
'snippet' => mb_substr($snip, 0, VV_AI_WEB_MAX_SNIPPET)];
|
|
}
|
|
return $out;
|
|
}
|
|
|
|
// ── Providers ────────────────────────────────────────────────────────────────────────────────
|
|
// Each returns raw {title,url,snippet} rows or null when the call failed. Only the one this
|
|
// installation is configured for has ever been exercised against a live service; the other two
|
|
// are written from their published response shapes and are unverified here. Whichever is in use,
|
|
// a shape that changes underneath produces zero results rather than wrong ones, because every
|
|
// field is read by name and anything missing is dropped by vv_ai_web_normalise().
|
|
|
|
// Degoog — self-hosted, no key, and the only provider here whose response has been read from a
|
|
// live service rather than from documentation. It aggregates several engines and returns them
|
|
// already merged, so the ranking is its own.
|
|
//
|
|
// snippet before content: it fills both, and around one result in six has neither. Those are kept
|
|
// anyway — a title and a URL is still a citable source, and dropping them would silently shrink
|
|
// the result set the operator asked for.
|
|
function vv_ai_web_degoog(string $q): ?array {
|
|
$base = vv_ai_web_degoog_url();
|
|
if ($base === '') return null;
|
|
$d = vv_ai_web_http($base . '/api/search?q=' . rawurlencode($q));
|
|
if ($d === null) return null;
|
|
|
|
// Its Reddit engine returns subreddit front pages for any query at all — "Rocket League
|
|
// Esports" and "Dividend Investing" came back for a question about ZFS scrubs, interleaved
|
|
// one in two, so half of what reached the model was noise. Reddit *threads* from the same
|
|
// engine are often the best result there is, so the engine is not excluded; the shape of URL
|
|
// that cannot contain an answer is.
|
|
//
|
|
// Structural, not a relevance judgement. /r/<name> with nothing after it is a community's
|
|
// front page by definition, which is a fact about the URL rather than an opinion about how
|
|
// well it matches — this file does no similarity scoring of its own and should not start.
|
|
$rows = [];
|
|
foreach ($d['results'] ?? [] as $r) {
|
|
$url = (string)($r['url'] ?? '');
|
|
// Delimited with ~, not #. The character class has to exclude a literal # (a fragment),
|
|
// and with # as the delimiter PCRE ends the pattern there and the match silently never
|
|
// fires — which is exactly what happened, and it looked like the filter doing nothing.
|
|
if (preg_match('~^https?://(?:[a-z0-9-]+\.)*reddit\.com/r/[^/?#]+/?$~i', $url)) continue;
|
|
$rows[] = ['title' => $r['title'] ?? '',
|
|
'url' => $url,
|
|
'snippet' => ($r['snippet'] ?? '') !== '' ? $r['snippet'] : ($r['content'] ?? '')];
|
|
if (count($rows) >= vv_ai_web_results_max()) break;
|
|
}
|
|
return $rows;
|
|
}
|
|
|
|
function vv_ai_web_searxng(string $q): ?array {
|
|
$base = vv_ai_web_searx_url();
|
|
if ($base === '') return null;
|
|
// format=json must be enabled in SearXNG's own settings.yml; it is off in the default image.
|
|
$url = $base . '/search?format=json&safesearch=0&q=' . rawurlencode($q);
|
|
$d = vv_ai_web_http($url);
|
|
if ($d === null) return null;
|
|
return array_map(fn($r) => ['title' => $r['title'] ?? '', 'url' => $r['url'] ?? '',
|
|
'snippet' => $r['content'] ?? ''],
|
|
array_slice($d['results'] ?? [], 0, vv_ai_web_results_max()));
|
|
}
|
|
|
|
function vv_ai_web_brave(string $q): ?array {
|
|
$key = vv_ai_web_api_key();
|
|
if ($key === '') return null;
|
|
$url = 'https://api.search.brave.com/res/v1/web/search?count=' . vv_ai_web_results_max()
|
|
. '&q=' . rawurlencode($q);
|
|
$d = vv_ai_web_http($url, ['X-Subscription-Token: ' . $key]);
|
|
if ($d === null) return null;
|
|
return array_map(fn($r) => ['title' => $r['title'] ?? '', 'url' => $r['url'] ?? '',
|
|
'snippet' => $r['description'] ?? ''],
|
|
array_slice($d['web']['results'] ?? [], 0, vv_ai_web_results_max()));
|
|
}
|
|
|
|
function vv_ai_web_tavily(string $q): ?array {
|
|
$key = vv_ai_web_api_key();
|
|
if ($key === '') return null;
|
|
$d = vv_ai_web_http('https://api.tavily.com/search', [],
|
|
['api_key' => $key, 'query' => $q, 'max_results' => vv_ai_web_results_max(),
|
|
'search_depth' => 'basic']);
|
|
if ($d === null) return null;
|
|
return array_map(fn($r) => ['title' => $r['title'] ?? '', 'url' => $r['url'] ?? '',
|
|
'snippet' => $r['content'] ?? ''],
|
|
array_slice($d['results'] ?? [], 0, vv_ai_web_results_max()));
|
|
}
|
|
|
|
// The one function the worker calls.
|
|
//
|
|
// Returns ['ok'=>bool, 'results'=>[…], 'provider'=>…, 'error'=>…]. An empty result set is not an
|
|
// error: "the web had nothing" and "the search did not happen" are different things to tell the
|
|
// model, and collapsing them is how an answer ends up asserting that nothing exists.
|
|
function vv_ai_web_search(string $question): array {
|
|
$ready = vv_ai_web_ready();
|
|
if (!($ready['ok'] ?? false)) {
|
|
return ['ok' => false, 'results' => [], 'provider' => vv_ai_web_provider(),
|
|
'error' => $ready['error']];
|
|
}
|
|
|
|
$q = trim(preg_replace('/\s+/u', ' ', $question));
|
|
if ($q === '') return ['ok' => false, 'results' => [], 'provider' => vv_ai_web_provider(),
|
|
'error' => 'empty query'];
|
|
$q = mb_substr($q, 0, VV_AI_WEB_MAX_QUERY);
|
|
|
|
$provider = vv_ai_web_provider();
|
|
$raw = match ($provider) {
|
|
'brave' => vv_ai_web_brave($q),
|
|
'tavily' => vv_ai_web_tavily($q),
|
|
'searxng' => vv_ai_web_searxng($q),
|
|
default => vv_ai_web_degoog($q),
|
|
};
|
|
|
|
if ($raw === null) {
|
|
return ['ok' => false, 'results' => [], 'provider' => $provider,
|
|
'error' => $provider . ' did not answer'];
|
|
}
|
|
return ['ok' => true, 'provider' => $provider,
|
|
'results' => array_slice(vv_ai_web_normalise($raw), 0, vv_ai_web_results_max())];
|
|
}
|
|
|
|
// The block that goes in front of the model, and the sources the page renders beside the answer.
|
|
// Numbered from an offset so web results can sit after retrieved passages without either set
|
|
// renumbering the other.
|
|
function vv_ai_web_context(array $results, int $offset = 0): string {
|
|
if (!$results) return '';
|
|
$s = "Web search results. These are from outside this installation — cite them by number, and\n"
|
|
. "say when something is a general answer rather than one about this machine.\n\n";
|
|
foreach ($results as $i => $r) {
|
|
$s .= '[' . ($offset + $i + 1) . '] ' . $r['title'] . ' — ' . $r['url'] . "\n"
|
|
. $r['snippet'] . "\n\n";
|
|
}
|
|
return $s;
|
|
}
|