Gmer4Lfe
07f679b2e7
Its Reddit engine returns a community's landing page for any query at all — a question about ZFS scrubs came back with Rocket League Esports and Dividend Investing, interleaved one in two, so half of what reached the model was noise. Threads from that same engine are frequently the best result there is, so the engine stays and only the URL shape goes: /r/<name> with nothing after it is a front page by definition. That is a fact about the URL, not an opinion about how well it matches, and this file does no similarity scoring of its own.
282 lines
14 KiB
PHP
282 lines
14 KiB
PHP
<?php
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
// PURPOSE
|
|
// Web search for the chat profile. Turns a question into a handful of titled snippets with
|
|
// their URLs, for the worker to put in front of the model as context it can cite.
|
|
//
|
|
// OPERATIONAL MODEL
|
|
// One provider at a time, named by AI_WEB_SEARCH_PROVIDER. Four are supported because the right
|
|
// answer depends on what the operator already runs: degoog and SearXNG are self-hosted and need
|
|
// no key and no third party, while brave and tavily need no container. They differ only in how
|
|
// the request is shaped and where the fields sit in the reply, so everything after parsing is
|
|
// one code path.
|
|
//
|
|
// degoog is the default and the one verified here — it was already running on this host, and it
|
|
// aggregates several engines and returns them merged. DuckDuckGo is deliberately absent: both
|
|
// its scraping endpoints answer a challenge page from this network, and its keyless API returns
|
|
// nothing at all for a real question.
|
|
//
|
|
// DESIGN PRINCIPLES
|
|
// Off by default, and not because it is dangerous.
|
|
// Searching sends the operator's question to something outside this house. That is a
|
|
// decision for the person whose question it is, so AI_WEB_SEARCH_ENABLED starts false and
|
|
// nothing turns it on.
|
|
//
|
|
// Chat only, and that is a capability, not a check in here.
|
|
// General chat holds no other capability precisely because it cannot write anything. Search
|
|
// is a read, so it is the one capability that fits there. The Varaverk assistant does not
|
|
// get it: its whole contract is that answers come from this installation's own documents,
|
|
// and a web result in that context is an answer that looks sourced and is not.
|
|
//
|
|
// Snippets, never pages.
|
|
// Only what the search API returns. Fetching the pages themselves would mean running an
|
|
// HTML parser over whatever the internet handed back, on a box that already has enough to
|
|
// do, for text the model has a 16k window to hold.
|
|
//
|
|
// OPERATIONAL SAFEGUARDS
|
|
// Bounded everywhere. A timeout on the call, a cap on the number of results, and a cap on the
|
|
// length of each snippet — the context this competes for is the same one retrieval needs.
|
|
//
|
|
// The query is the operator's question and nothing else. No conf values, no host names, no log
|
|
// contents are ever appended to it: this is the one path in the plugin that sends text off the
|
|
// machine, and what leaves has to be exactly what the operator typed.
|
|
//
|
|
// Redacted before it is logged, like every other question in this subsystem.
|
|
//
|
|
// DEPENDS ON
|
|
// include/ai.php vv_ai_redact(), vv_conf_vars()
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
require_once __DIR__ . '/ai.php';
|
|
|
|
const VV_AI_WEB_PROVIDERS = ['degoog', 'searxng', 'brave', 'tavily'];
|
|
const VV_AI_WEB_MAX_QUERY = 300; // characters sent to the provider
|
|
const VV_AI_WEB_MAX_SNIPPET = 400; // characters kept from each result
|
|
|
|
function vv_ai_web_enabled(): bool {
|
|
if (!vv_ai_enabled()) return false;
|
|
return strtolower(trim((string)(vv_conf_vars()['AI_WEB_SEARCH_ENABLED'] ?? 'false'))) === 'true';
|
|
}
|
|
|
|
function vv_ai_web_provider(): string {
|
|
$p = strtolower(trim((string)(vv_conf_vars()['AI_WEB_SEARCH_PROVIDER'] ?? 'degoog')));
|
|
return in_array($p, VV_AI_WEB_PROVIDERS, true) ? $p : 'degoog';
|
|
}
|
|
|
|
function vv_ai_web_results_max(): int {
|
|
return max(1, min(10, (int)(vv_conf_vars()['AI_WEB_SEARCH_RESULTS'] ?? 4)));
|
|
}
|
|
|
|
function vv_ai_web_timeout(): int {
|
|
return max(2, min(30, (int)(vv_conf_vars()['AI_WEB_SEARCH_TIMEOUT'] ?? 6)));
|
|
}
|
|
|
|
// Per-host, because a SearXNG URL is an address on this network and an API key is a credential —
|
|
// both belong in host*.conf by the same rule that puts every other one there.
|
|
function vv_ai_web_searx_url(): string {
|
|
return rtrim(trim((string)(vv_conf_vars()[strtoupper(vv_detect_host()) . '_SEARXNG_URL'] ?? '')), '/');
|
|
}
|
|
|
|
function vv_ai_web_degoog_url(): string {
|
|
return rtrim(trim((string)(vv_conf_vars()[strtoupper(vv_detect_host()) . '_DEGOOG_URL'] ?? '')), '/');
|
|
}
|
|
|
|
// Which providers need an address on this network rather than a key. Kept as a list because the
|
|
// readiness check and the error message both need the same answer and disagreeing about it is how
|
|
// a feature reports itself ready and then does nothing.
|
|
const VV_AI_WEB_SELF_HOSTED = ['degoog', 'searxng'];
|
|
|
|
function vv_ai_web_api_key(): string {
|
|
return trim((string)(vv_conf_vars()[strtoupper(vv_detect_host()) . '_WEB_SEARCH_API_KEY'] ?? ''));
|
|
}
|
|
|
|
// Is this provider actually set up? Answered separately from "is it switched on", because the
|
|
// two failures need different sentences: one is a setting, the other is a missing address or key.
|
|
function vv_ai_web_ready(): array {
|
|
if (!vv_ai_web_enabled()) {
|
|
return ['ok' => false, 'error' => 'AI_WEB_SEARCH_ENABLED is false'];
|
|
}
|
|
$p = vv_ai_web_provider();
|
|
$host = strtoupper(vv_detect_host());
|
|
|
|
if ($p === 'degoog' && vv_ai_web_degoog_url() === '') {
|
|
return ['ok' => false, 'error' => $host . '_DEGOOG_URL is empty'];
|
|
}
|
|
if ($p === 'searxng' && vv_ai_web_searx_url() === '') {
|
|
return ['ok' => false, 'error' => $host . '_SEARXNG_URL is empty'];
|
|
}
|
|
if (!in_array($p, VV_AI_WEB_SELF_HOSTED, true) && vv_ai_web_api_key() === '') {
|
|
return ['ok' => false, 'error' => $host . '_WEB_SEARCH_API_KEY is empty'];
|
|
}
|
|
return ['ok' => true, 'provider' => $p];
|
|
}
|
|
|
|
// One HTTP call, returning the decoded body or null. curl rather than file_get_contents so the
|
|
// timeout is real and a proxy-less environment does not hang the worker.
|
|
function vv_ai_web_http(string $url, array $headers = [], ?array $postJson = null): ?array {
|
|
$ch = curl_init($url);
|
|
if ($ch === false) return null;
|
|
curl_setopt_array($ch, [
|
|
CURLOPT_RETURNTRANSFER => true,
|
|
CURLOPT_TIMEOUT => vv_ai_web_timeout(),
|
|
CURLOPT_CONNECTTIMEOUT => min(5, vv_ai_web_timeout()),
|
|
CURLOPT_FOLLOWLOCATION => false,
|
|
CURLOPT_HTTPHEADER => array_merge(['Accept: application/json'], $headers),
|
|
CURLOPT_USERAGENT => 'Varaverk',
|
|
]);
|
|
if ($postJson !== null) {
|
|
curl_setopt($ch, CURLOPT_POST, true);
|
|
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($postJson));
|
|
curl_setopt($ch, CURLOPT_HTTPHEADER,
|
|
array_merge(['Accept: application/json', 'Content-Type: application/json'], $headers));
|
|
}
|
|
$body = curl_exec($ch);
|
|
$code = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
|
curl_close($ch);
|
|
if ($body === false || $code < 200 || $code >= 300) return null;
|
|
$d = json_decode((string)$body, true);
|
|
return is_array($d) ? $d : null;
|
|
}
|
|
|
|
// Everything below reduces to the same three fields, so the worker never learns which provider
|
|
// answered: title, url, snippet.
|
|
function vv_ai_web_normalise(array $rows): array {
|
|
$out = [];
|
|
foreach ($rows as $r) {
|
|
$url = trim((string)($r['url'] ?? ''));
|
|
// Only http(s). A provider returning anything else is either broken or hostile, and this
|
|
// string ends up in an href.
|
|
if (!preg_match('#^https?://#i', $url)) continue;
|
|
$title = trim((string)($r['title'] ?? ''));
|
|
$snip = trim(preg_replace('/\s+/u', ' ', (string)($r['snippet'] ?? '')));
|
|
if ($title === '' && $snip === '') continue;
|
|
$out[] = ['title' => mb_substr($title !== '' ? $title : $url, 0, 160),
|
|
'url' => $url,
|
|
'snippet' => mb_substr($snip, 0, VV_AI_WEB_MAX_SNIPPET)];
|
|
}
|
|
return $out;
|
|
}
|
|
|
|
// ── Providers ────────────────────────────────────────────────────────────────────────────────
|
|
// Each returns raw {title,url,snippet} rows or null when the call failed. Only the one this
|
|
// installation is configured for has ever been exercised against a live service; the other two
|
|
// are written from their published response shapes and are unverified here. Whichever is in use,
|
|
// a shape that changes underneath produces zero results rather than wrong ones, because every
|
|
// field is read by name and anything missing is dropped by vv_ai_web_normalise().
|
|
|
|
// Degoog — self-hosted, no key, and the only provider here whose response has been read from a
|
|
// live service rather than from documentation. It aggregates several engines and returns them
|
|
// already merged, so the ranking is its own.
|
|
//
|
|
// snippet before content: it fills both, and around one result in six has neither. Those are kept
|
|
// anyway — a title and a URL is still a citable source, and dropping them would silently shrink
|
|
// the result set the operator asked for.
|
|
function vv_ai_web_degoog(string $q): ?array {
|
|
$base = vv_ai_web_degoog_url();
|
|
if ($base === '') return null;
|
|
$d = vv_ai_web_http($base . '/api/search?q=' . rawurlencode($q));
|
|
if ($d === null) return null;
|
|
|
|
// Its Reddit engine returns subreddit front pages for any query at all — "Rocket League
|
|
// Esports" and "Dividend Investing" came back for a question about ZFS scrubs, interleaved
|
|
// one in two, so half of what reached the model was noise. Reddit *threads* from the same
|
|
// engine are often the best result there is, so the engine is not excluded; the shape of URL
|
|
// that cannot contain an answer is.
|
|
//
|
|
// Structural, not a relevance judgement. /r/<name> with nothing after it is a community's
|
|
// front page by definition, which is a fact about the URL rather than an opinion about how
|
|
// well it matches — this file does no similarity scoring of its own and should not start.
|
|
$rows = [];
|
|
foreach ($d['results'] ?? [] as $r) {
|
|
$url = (string)($r['url'] ?? '');
|
|
if (preg_match('#^https?://(?:[a-z0-9-]+\.)*reddit\.com/r/[^/?#]+/?$#i', $url)) continue;
|
|
$rows[] = ['title' => $r['title'] ?? '',
|
|
'url' => $url,
|
|
'snippet' => ($r['snippet'] ?? '') !== '' ? $r['snippet'] : ($r['content'] ?? '')];
|
|
if (count($rows) >= vv_ai_web_results_max()) break;
|
|
}
|
|
return $rows;
|
|
}
|
|
|
|
function vv_ai_web_searxng(string $q): ?array {
|
|
$base = vv_ai_web_searx_url();
|
|
if ($base === '') return null;
|
|
// format=json must be enabled in SearXNG's own settings.yml; it is off in the default image.
|
|
$url = $base . '/search?format=json&safesearch=0&q=' . rawurlencode($q);
|
|
$d = vv_ai_web_http($url);
|
|
if ($d === null) return null;
|
|
return array_map(fn($r) => ['title' => $r['title'] ?? '', 'url' => $r['url'] ?? '',
|
|
'snippet' => $r['content'] ?? ''],
|
|
array_slice($d['results'] ?? [], 0, vv_ai_web_results_max()));
|
|
}
|
|
|
|
function vv_ai_web_brave(string $q): ?array {
|
|
$key = vv_ai_web_api_key();
|
|
if ($key === '') return null;
|
|
$url = 'https://api.search.brave.com/res/v1/web/search?count=' . vv_ai_web_results_max()
|
|
. '&q=' . rawurlencode($q);
|
|
$d = vv_ai_web_http($url, ['X-Subscription-Token: ' . $key]);
|
|
if ($d === null) return null;
|
|
return array_map(fn($r) => ['title' => $r['title'] ?? '', 'url' => $r['url'] ?? '',
|
|
'snippet' => $r['description'] ?? ''],
|
|
array_slice($d['web']['results'] ?? [], 0, vv_ai_web_results_max()));
|
|
}
|
|
|
|
function vv_ai_web_tavily(string $q): ?array {
|
|
$key = vv_ai_web_api_key();
|
|
if ($key === '') return null;
|
|
$d = vv_ai_web_http('https://api.tavily.com/search', [],
|
|
['api_key' => $key, 'query' => $q, 'max_results' => vv_ai_web_results_max(),
|
|
'search_depth' => 'basic']);
|
|
if ($d === null) return null;
|
|
return array_map(fn($r) => ['title' => $r['title'] ?? '', 'url' => $r['url'] ?? '',
|
|
'snippet' => $r['content'] ?? ''],
|
|
array_slice($d['results'] ?? [], 0, vv_ai_web_results_max()));
|
|
}
|
|
|
|
// The one function the worker calls.
|
|
//
|
|
// Returns ['ok'=>bool, 'results'=>[…], 'provider'=>…, 'error'=>…]. An empty result set is not an
|
|
// error: "the web had nothing" and "the search did not happen" are different things to tell the
|
|
// model, and collapsing them is how an answer ends up asserting that nothing exists.
|
|
function vv_ai_web_search(string $question): array {
|
|
$ready = vv_ai_web_ready();
|
|
if (!($ready['ok'] ?? false)) {
|
|
return ['ok' => false, 'results' => [], 'provider' => vv_ai_web_provider(),
|
|
'error' => $ready['error']];
|
|
}
|
|
|
|
$q = trim(preg_replace('/\s+/u', ' ', $question));
|
|
if ($q === '') return ['ok' => false, 'results' => [], 'provider' => vv_ai_web_provider(),
|
|
'error' => 'empty query'];
|
|
$q = mb_substr($q, 0, VV_AI_WEB_MAX_QUERY);
|
|
|
|
$provider = vv_ai_web_provider();
|
|
$raw = match ($provider) {
|
|
'brave' => vv_ai_web_brave($q),
|
|
'tavily' => vv_ai_web_tavily($q),
|
|
'searxng' => vv_ai_web_searxng($q),
|
|
default => vv_ai_web_degoog($q),
|
|
};
|
|
|
|
if ($raw === null) {
|
|
return ['ok' => false, 'results' => [], 'provider' => $provider,
|
|
'error' => $provider . ' did not answer'];
|
|
}
|
|
return ['ok' => true, 'provider' => $provider,
|
|
'results' => array_slice(vv_ai_web_normalise($raw), 0, vv_ai_web_results_max())];
|
|
}
|
|
|
|
// The block that goes in front of the model, and the sources the page renders beside the answer.
|
|
// Numbered from an offset so web results can sit after retrieved passages without either set
|
|
// renumbering the other.
|
|
function vv_ai_web_context(array $results, int $offset = 0): string {
|
|
if (!$results) return '';
|
|
$s = "Web search results. These are from outside this installation — cite them by number, and\n"
|
|
. "say when something is a general answer rather than one about this machine.\n\n";
|
|
foreach ($results as $i => $r) {
|
|
$s .= '[' . ($offset + $i + 1) . '] ' . $r['title'] . ' — ' . $r['url'] . "\n"
|
|
. $r['snippet'] . "\n\n";
|
|
}
|
|
return $s;
|
|
}
|