<?php
// ═══════════════════════════════════════════════════════════════════════════════════════════════
// PURPOSE
//   Web search for the chat profile. Turns a question into a handful of titled snippets with
//   their URLs, for the worker to put in front of the model as context it can cite.
//
// OPERATIONAL MODEL
//   One provider at a time, named by AI_WEB_SEARCH_PROVIDER. Four are supported because the right
//   answer depends on what the operator already runs: degoog and SearXNG are self-hosted and need
//   no key and no third party, while brave and tavily need no container. They differ only in how
//   the request is shaped and where the fields sit in the reply, so everything after parsing is
//   one code path.
//
//   degoog is the default and the one verified here — it was already running on this host, and it
//   aggregates several engines and returns them merged. DuckDuckGo is deliberately absent: both
//   its scraping endpoints answer a challenge page from this network, and its keyless API returns
//   nothing at all for a real question.
//
// DESIGN PRINCIPLES
//   Off by default, and not because it is dangerous.
//       Searching sends the operator's question to something outside this house. That is a
//       decision for the person whose question it is, so AI_WEB_SEARCH_ENABLED starts false and
//       nothing turns it on.
//
//   Chat only, and that is a capability, not a check in here.
//       General chat holds no other capability precisely because it cannot write anything. Search
//       is a read, so it is the one capability that fits there. The Varaverk assistant does not
//       get it: its whole contract is that answers come from this installation's own documents,
//       and a web result in that context is an answer that looks sourced and is not.
//
//   Snippets, never pages.
//       Only what the search API returns. Fetching the pages themselves would mean running an
//       HTML parser over whatever the internet handed back, on a box that already has enough to
//       do, for text the model has a 16k window to hold.
//
// OPERATIONAL SAFEGUARDS
//   Bounded everywhere. A timeout on the call, a cap on the number of results, and a cap on the
//   length of each snippet — the context this competes for is the same one retrieval needs.
//
//   The query is the operator's question and nothing else. No conf values, no host names, no log
//   contents are ever appended to it: this is the one path in the plugin that sends text off the
//   machine, and what leaves has to be exactly what the operator typed.
//
//   Redacted before it is logged, like every other question in this subsystem.
//
// EXPORTS
//   Settings   vv_ai_web_enabled(), vv_ai_web_provider(), vv_ai_web_results_max(),
//              vv_ai_web_timeout(), vv_ai_web_searx_url(), vv_ai_web_degoog_url(),
//              vv_ai_web_api_key(), vv_ai_web_ready()
//              — every one reads conf; none of them reaches the network.
//   Providers  vv_ai_web_degoog(), vv_ai_web_searxng(), vv_ai_web_brave(), vv_ai_web_tavily()
//              — one per backend, each returning the same normalised shape.
//   Transport  vv_ai_web_http(), vv_ai_web_normalise()
//   Entry      vv_ai_web_search(), vv_ai_web_context()
//              — what the rest of the subsystem calls. Everything above is reachable but only
//              these two are meant to be: search() picks the provider, context() formats for
//              the prompt.
//
// CONFIGURATION
//   master.conf
//     AI_WEB_SEARCH_ENABLED    master switch; anything but "true" disables the whole subsystem
//     AI_WEB_SEARCH_PROVIDER   which backend vv_ai_web_search() dispatches to
//     AI_WEB_SEARCH_RESULTS    result cap handed to the provider
//     AI_WEB_SEARCH_TIMEOUT    per-request timeout, in seconds
//
//   host*.conf — per host, because the endpoint and the key are this machine's
//     <HOST>_SEARXNG_URL         self-hosted SearXNG base URL
//     <HOST>_DEGOOG_URL          self-hosted degoogle/Whoogle base URL
//     <HOST>_WEB_SEARCH_API_KEY  Brave or Tavily key, whichever provider is selected
//
// DEPENDS ON
//   include/ai.php   vv_ai_redact(), vv_conf_vars()
// ═══════════════════════════════════════════════════════════════════════════════════════════════
require_once __DIR__ . '/ai.php';

const VV_AI_WEB_PROVIDERS   = ['degoog', 'searxng', 'brave', 'tavily'];
const VV_AI_WEB_MAX_QUERY   = 300;   // characters sent to the provider
const VV_AI_WEB_MAX_SNIPPET = 400;   // characters kept from each result

function vv_ai_web_enabled(): bool {
    if (!vv_ai_enabled()) return false;
    return strtolower(trim((string)(vv_conf_vars()['AI_WEB_SEARCH_ENABLED'] ?? 'false'))) === 'true';
}

function vv_ai_web_provider(): string {
    $p = strtolower(trim((string)(vv_conf_vars()['AI_WEB_SEARCH_PROVIDER'] ?? 'degoog')));
    return in_array($p, VV_AI_WEB_PROVIDERS, true) ? $p : 'degoog';
}

function vv_ai_web_results_max(): int {
    return max(1, min(10, (int)(vv_conf_vars()['AI_WEB_SEARCH_RESULTS'] ?? 4)));
}

function vv_ai_web_timeout(): int {
    return max(2, min(30, (int)(vv_conf_vars()['AI_WEB_SEARCH_TIMEOUT'] ?? 6)));
}

// Per-host, because a SearXNG URL is an address on this network and an API key is a credential —
// both belong in host*.conf by the same rule that puts every other one there.
function vv_ai_web_searx_url(): string {
    return rtrim(trim((string)(vv_conf_vars()[strtoupper(vv_detect_host()) . '_SEARXNG_URL'] ?? '')), '/');
}

function vv_ai_web_degoog_url(): string {
    return rtrim(trim((string)(vv_conf_vars()[strtoupper(vv_detect_host()) . '_DEGOOG_URL'] ?? '')), '/');
}

// Which providers need an address on this network rather than a key. Kept as a list because the
// readiness check and the error message both need the same answer and disagreeing about it is how
// a feature reports itself ready and then does nothing.
const VV_AI_WEB_SELF_HOSTED = ['degoog', 'searxng'];

function vv_ai_web_api_key(): string {
    return trim((string)(vv_conf_vars()[strtoupper(vv_detect_host()) . '_WEB_SEARCH_API_KEY'] ?? ''));
}

// Is this provider actually set up? Answered separately from "is it switched on", because the
// two failures need different sentences: one is a setting, the other is a missing address or key.
function vv_ai_web_ready(): array {
    if (!vv_ai_web_enabled()) {
        return ['ok' => false, 'error' => 'AI_WEB_SEARCH_ENABLED is false'];
    }
    $p    = vv_ai_web_provider();
    $host = strtoupper(vv_detect_host());

    if ($p === 'degoog' && vv_ai_web_degoog_url() === '') {
        return ['ok' => false, 'error' => $host . '_DEGOOG_URL is empty'];
    }
    if ($p === 'searxng' && vv_ai_web_searx_url() === '') {
        return ['ok' => false, 'error' => $host . '_SEARXNG_URL is empty'];
    }
    if (!in_array($p, VV_AI_WEB_SELF_HOSTED, true) && vv_ai_web_api_key() === '') {
        return ['ok' => false, 'error' => $host . '_WEB_SEARCH_API_KEY is empty'];
    }
    return ['ok' => true, 'provider' => $p];
}

// One HTTP call, returning the decoded body or null. curl rather than file_get_contents so the
// timeout is real and a proxy-less environment does not hang the worker.
function vv_ai_web_http(string $url, array $headers = [], ?array $postJson = null): ?array {
    $ch = curl_init($url);
    if ($ch === false) return null;
    curl_setopt_array($ch, [
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_TIMEOUT        => vv_ai_web_timeout(),
        CURLOPT_CONNECTTIMEOUT => min(5, vv_ai_web_timeout()),
        CURLOPT_FOLLOWLOCATION => false,
        CURLOPT_HTTPHEADER     => array_merge(['Accept: application/json'], $headers),
        CURLOPT_USERAGENT      => 'Varaverk',
    ]);
    if ($postJson !== null) {
        curl_setopt($ch, CURLOPT_POST, true);
        curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($postJson));
        curl_setopt($ch, CURLOPT_HTTPHEADER,
            array_merge(['Accept: application/json', 'Content-Type: application/json'], $headers));
    }
    $body = curl_exec($ch);
    $code = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
    curl_close($ch);
    if ($body === false || $code < 200 || $code >= 300) return null;
    $d = json_decode((string)$body, true);
    return is_array($d) ? $d : null;
}

// Everything below reduces to the same three fields, so the worker never learns which provider
// answered: title, url, snippet.
function vv_ai_web_normalise(array $rows): array {
    $out = [];
    foreach ($rows as $r) {
        $url = trim((string)($r['url'] ?? ''));
        // Only http(s). A provider returning anything else is either broken or hostile, and this
        // string ends up in an href.
        if (!preg_match('#^https?://#i', $url)) continue;
        $title = trim((string)($r['title'] ?? ''));
        $snip  = trim(preg_replace('/\s+/u', ' ', (string)($r['snippet'] ?? '')));
        if ($title === '' && $snip === '') continue;
        $out[] = ['title'   => mb_substr($title !== '' ? $title : $url, 0, 160),
                  'url'     => $url,
                  'snippet' => mb_substr($snip, 0, VV_AI_WEB_MAX_SNIPPET)];
    }
    return $out;
}

// ── Providers ────────────────────────────────────────────────────────────────────────────────
// Each returns raw {title,url,snippet} rows or null when the call failed. Only the one this
// installation is configured for has ever been exercised against a live service; the other two
// are written from their published response shapes and are unverified here. Whichever is in use,
// a shape that changes underneath produces zero results rather than wrong ones, because every
// field is read by name and anything missing is dropped by vv_ai_web_normalise().

// Degoog — self-hosted, no key, and the only provider here whose response has been read from a
// live service rather than from documentation. It aggregates several engines and returns them
// already merged, so the ranking is its own.
//
// snippet before content: it fills both, and around one result in six has neither. Those are kept
// anyway — a title and a URL is still a citable source, and dropping them would silently shrink
// the result set the operator asked for.
function vv_ai_web_degoog(string $q): ?array {
    $base = vv_ai_web_degoog_url();
    if ($base === '') return null;
    $d = vv_ai_web_http($base . '/api/search?q=' . rawurlencode($q));
    if ($d === null) return null;

    // Its Reddit engine returns subreddit front pages for any query at all — "Rocket League
    // Esports" and "Dividend Investing" came back for a question about ZFS scrubs, interleaved
    // one in two, so half of what reached the model was noise. Reddit *threads* from the same
    // engine are often the best result there is, so the engine is not excluded; the shape of URL
    // that cannot contain an answer is.
    //
    // Structural, not a relevance judgement. /r/<name> with nothing after it is a community's
    // front page by definition, which is a fact about the URL rather than an opinion about how
    // well it matches — this file does no similarity scoring of its own and should not start.
    $rows = [];
    foreach ($d['results'] ?? [] as $r) {
        $url = (string)($r['url'] ?? '');
        // Delimited with ~, not #. The character class has to exclude a literal # (a fragment),
        // and with # as the delimiter PCRE ends the pattern there and the match silently never
        // fires — which is exactly what happened, and it looked like the filter doing nothing.
        if (preg_match('~^https?://(?:[a-z0-9-]+\.)*reddit\.com/r/[^/?#]+/?$~i', $url)) continue;
        $rows[] = ['title'   => $r['title'] ?? '',
                   'url'     => $url,
                   'snippet' => ($r['snippet'] ?? '') !== '' ? $r['snippet'] : ($r['content'] ?? '')];
        if (count($rows) >= vv_ai_web_results_max()) break;
    }
    return $rows;
}

function vv_ai_web_searxng(string $q): ?array {
    $base = vv_ai_web_searx_url();
    if ($base === '') return null;
    // format=json must be enabled in SearXNG's own settings.yml; it is off in the default image.
    $url = $base . '/search?format=json&safesearch=0&q=' . rawurlencode($q);
    $d   = vv_ai_web_http($url);
    if ($d === null) return null;
    return array_map(fn($r) => ['title' => $r['title'] ?? '', 'url' => $r['url'] ?? '',
                                'snippet' => $r['content'] ?? ''],
                     array_slice($d['results'] ?? [], 0, vv_ai_web_results_max()));
}

function vv_ai_web_brave(string $q): ?array {
    $key = vv_ai_web_api_key();
    if ($key === '') return null;
    $url = 'https://api.search.brave.com/res/v1/web/search?count=' . vv_ai_web_results_max()
         . '&q=' . rawurlencode($q);
    $d = vv_ai_web_http($url, ['X-Subscription-Token: ' . $key]);
    if ($d === null) return null;
    return array_map(fn($r) => ['title' => $r['title'] ?? '', 'url' => $r['url'] ?? '',
                                'snippet' => $r['description'] ?? ''],
                     array_slice($d['web']['results'] ?? [], 0, vv_ai_web_results_max()));
}

function vv_ai_web_tavily(string $q): ?array {
    $key = vv_ai_web_api_key();
    if ($key === '') return null;
    $d = vv_ai_web_http('https://api.tavily.com/search', [],
        ['api_key' => $key, 'query' => $q, 'max_results' => vv_ai_web_results_max(),
         'search_depth' => 'basic']);
    if ($d === null) return null;
    return array_map(fn($r) => ['title' => $r['title'] ?? '', 'url' => $r['url'] ?? '',
                                'snippet' => $r['content'] ?? ''],
                     array_slice($d['results'] ?? [], 0, vv_ai_web_results_max()));
}

// The one function the worker calls.
//
// Returns ['ok'=>bool, 'results'=>[…], 'provider'=>…, 'error'=>…]. An empty result set is not an
// error: "the web had nothing" and "the search did not happen" are different things to tell the
// model, and collapsing them is how an answer ends up asserting that nothing exists.
function vv_ai_web_search(string $question): array {
    $ready = vv_ai_web_ready();
    if (!($ready['ok'] ?? false)) {
        return ['ok' => false, 'results' => [], 'provider' => vv_ai_web_provider(),
                'error' => $ready['error']];
    }

    $q = trim(preg_replace('/\s+/u', ' ', $question));
    if ($q === '') return ['ok' => false, 'results' => [], 'provider' => vv_ai_web_provider(),
                           'error' => 'empty query'];
    $q = mb_substr($q, 0, VV_AI_WEB_MAX_QUERY);

    $provider = vv_ai_web_provider();
    $raw = match ($provider) {
        'brave'   => vv_ai_web_brave($q),
        'tavily'  => vv_ai_web_tavily($q),
        'searxng' => vv_ai_web_searxng($q),
        default   => vv_ai_web_degoog($q),
    };

    if ($raw === null) {
        return ['ok' => false, 'results' => [], 'provider' => $provider,
                'error' => $provider . ' did not answer'];
    }
    return ['ok' => true, 'provider' => $provider,
            'results' => array_slice(vv_ai_web_normalise($raw), 0, vv_ai_web_results_max())];
}

// The block that goes in front of the model, and the sources the page renders beside the answer.
// Numbered from an offset so web results can sit after retrieved passages without either set
// renumbering the other.
function vv_ai_web_context(array $results, int $offset = 0): string {
    if (!$results) return '';
    // Framed as reference material, not as the answer. The search runs whenever the box is
    // ticked, because a search is one HTTP request and deciding to run it would cost a whole
    // extra generation — but running it must not mean leaning on it. Answering a question the
    // model already knows by paraphrasing the first search result is worse than answering it
    // directly, and it was doing that because the results announced themselves as the source.
    $s = "Reference material from a web search, in case it is useful. It is from outside this\n"
       . "installation and nothing here has been verified against this machine.\n\n"
       . "Answer from what you already know when you know it. Reach for these when you do not,\n"
       . "when the question turns on something current, or when a specific version, release or\n"
       . "figure needs checking rather than recalling. Cite by number only what you actually\n"
       . "used, ignore the rest, and never imply the web was consulted about this installation —\n"
       . "it was not. Say plainly when an answer is general rather than about this machine.\n\n";
    foreach ($results as $i => $r) {
        $s .= '[' . ($offset + $i + 1) . '] ' . $r['title'] . ' — ' . $r['url'] . "\n"
            . $r['snippet'] . "\n\n";
    }
    return $s;
}