Add a retrieval-only HTTP bridge so Open-WebUI can reach the index

Open-WebUI runs in its own container with no WebGUI session, and nginx applies
auth_request to everything it serves, so it cannot call the plugin API. Same
shape as the arr webhook listener: node outside nginx, shared secret, one
read-only verb. Serves chunks rather than answers because the caller already
has a model loaded.

AI/ai_serve.js

@@ -0,0 +1,177 @@
+#!/usr/bin/env node
+// ═══════════════════════════════════════════════════════════════════════════════════════════════
+// PURPOSE
+//   Retrieval-only HTTP bridge over the AI index. Lets a client that cannot see Varaverk's
+//   filesystem — Open-WebUI in its own container — search the corpus and receive grounding
+//   chunks as JSON.
+//
+// OPERATIONAL MODEL
+//   Started by start_ai_server.sh, which is the only thing that knows the port and secret.
+//   Lives outside Unraid's nginx exactly as webhook_listener.js does, because the caller is a
+//   container with no WebGUI session and nginx applies auth_request to everything it serves.
+//   The shared secret in the query string is therefore the only gate.
+//
+//   Returns chunks, never answers. Open-WebUI already has a model loaded and is going to
+//   generate from whatever this returns; generating here as well would double the latency and
+//   burn a second model load for a result the caller discards. ai_query.sh keeps the
+//   generation path for CLI use — this endpoint deliberately does not.
+//
+//   Calls search() in-process rather than shelling out to ai_query.sh. A shell round trip per
+//   request would fork bash, re-source three conf files and re-open the index every time, for
+//   a call that is otherwise a single embed plus a vector scan.
+//
+// DESIGN PRINCIPLES
+//   One endpoint, one verb.
+//       GET /search. No index management, no conf access, no generation. The surface an
+//       unauthenticated port exposes should be the smallest thing that does the job.
+//
+//   The caller chooses breadth, within bounds.
+//       k, kind and section are accepted because the client knows what kind of question it is
+//       asking. All are clamped or validated here — the caller is trusted to express intent,
+//       not to be correct.
+//
+//   Failures answer in the same shape as successes.
+//       Every response is JSON with an ok flag. A tool calling this has no way to render an
+//       HTML error page, and a caller that cannot parse the failure reports nothing at all.
+//
+// OPERATIONAL SAFEGUARDS
+//   The secret is compared at fixed length before anything else runs.
+//       No index is opened and no embedding is requested until the key matches, so an
+//       unauthenticated caller cannot make this process do work. Compared with
+//       timingSafeEqual on equal-length buffers — a plain !== leaks length and position
+//       through response timing on a port that is, by design, reachable without a session.
+//
+//   Read-only by construction.
+//       The index is opened by search() for reading and nothing here writes to it, to conf,
+//       or to the filesystem. The worst a valid key can do is read documentation that is
+//       already in a git repository.
+//
+//   Bounded work per request.
+//       k is clamped to 1..25 and the query to 2000 characters, so no single call can pull the
+//       whole index into memory or embed an unbounded string.
+//
+//   Bad input is refused, not guessed.
+//       kind is checked against the five real values; an unknown one is rejected rather than
+//       passed to SQL where it would match zero rows and read as "the index has no answer" —
+//       the most misleading failure this system can produce.
+//
+//   Binds where it is told, and says so.
+//       The bind address comes from the launcher. It is reachable from the docker bridge by
+//       necessity; that is why the secret exists and why the surface is one read-only verb.
+//
+// CONFIGURATION
+//   argv: <port> <secret> <bind> <db> <ollamaUrl> <embedModel>
+//   All supplied by start_ai_server.sh from AI_HTTP_PORT, AI_HTTP_SECRET, AI_INDEX_DB and
+//   <HOST>_OLLAMA_URL / _OLLAMA_EMBED_MODEL.
+//
+// RUNTIME MODES
+//   GET /search?key=…&q=…[&k=8][&kind=readme][&section=PURPOSE]
+//   GET /health?key=…      liveness only — no index access
+// ═══════════════════════════════════════════════════════════════════════════════════════════════
+
+'use strict';
+
+const http   = require('http');
+const crypto = require('crypto');
+const path   = require('path');
+
+const [, , port, secret, bind, db, ollamaUrl, embedModel] = process.argv;
+
+if (!port || !secret || !bind || !db || !ollamaUrl || !embedModel) {
+    process.stderr.write('Usage: ai_serve.js <port> <secret> <bind> <db> <ollamaUrl> <embedModel>\n');
+    process.exit(1);
+}
+
+const { search } = require(path.join(__dirname, 'lib', 'search.js'));
+
+const KINDS    = new Set(['header', 'readme', 'manual', 'template', 'doc']);
+const MAX_Q    = 2000;
+const SECRET_B = Buffer.from(secret);
+
+function send(res, code, obj) {
+    const body = JSON.stringify(obj);
+    res.writeHead(code, {
+        'Content-Type':   'application/json',
+        'Content-Length': Buffer.byteLength(body),
+        'Cache-Control':  'no-store',
+    });
+    res.end(body);
+}
+
+// Equal-length compare first: timingSafeEqual throws on a length mismatch, and the length
+// itself is not worth leaking through an exception path on an unauthenticated port.
+function keyOk(given) {
+    if (typeof given !== 'string') return false;
+    const g = Buffer.from(given);
+    if (g.length !== SECRET_B.length) return false;
+    return crypto.timingSafeEqual(g, SECRET_B);
+}
+
+const server = http.createServer(async (req, res) => {
+    let url;
+    try { url = new URL(req.url, 'http://localhost'); }
+    catch { return send(res, 400, { ok: false, error: 'Bad request' }); }
+
+    if (!keyOk(url.searchParams.get('key'))) {
+        return send(res, 403, { ok: false, error: 'Forbidden' });
+    }
+
+    if (url.pathname === '/health') {
+        return send(res, 200, { ok: true, service: 'ai_serve', db });
+    }
+
+    if (url.pathname !== '/search') {
+        return send(res, 404, { ok: false, error: 'Not found' });
+    }
+
+    const q = (url.searchParams.get('q') || '').trim();
+    if (!q) return send(res, 400, { ok: false, error: 'q is required' });
+    if (q.length > MAX_Q) {
+        return send(res, 400, { ok: false, error: `q exceeds ${MAX_Q} characters` });
+    }
+
+    const kind = url.searchParams.get('kind') || null;
+    if (kind && !KINDS.has(kind)) {
+        return send(res, 400, {
+            ok: false,
+            error: `unknown kind '${kind}' (expected: ${[...KINDS].join(', ')})`,
+        });
+    }
+
+    let k = parseInt(url.searchParams.get('k') || '8', 10);
+    if (!Number.isFinite(k)) k = 8;
+    k = Math.min(Math.max(k, 1), 25);
+
+    try {
+        const r = await search({
+            dbPath: db, url: ollamaUrl, embedModel, query: q,
+            k, perFile: 3,
+            section: url.searchParams.get('section') || null,
+            kind,
+        });
+        send(res, 200, {
+            ok: true,
+            query:   q,
+            intents: r.intents,
+            scanned: r.scanned,
+            results: r.results.map(x => ({
+                path:    x.path,
+                section: x.section,
+                heading: x.heading,
+                score:   Number(x.score.toFixed(4)),
+                content: x.content,
+            })),
+        });
+    } catch (e) {
+        send(res, 500, { ok: false, error: `retrieval failed: ${e.message}` });
+    }
+});
+
+server.on('error', (e) => {
+    process.stderr.write(`ai_serve: ${e.message}\n`);
+    process.exit(1);
+});
+
+server.listen(parseInt(port, 10), bind, () => {
+    process.stdout.write(`ai_serve listening on ${bind}:${port}\n`);
+});

AI/start_ai_server.sh

@@ -0,0 +1,167 @@
+#!/bin/bash
+# ==============================================================================================
+# ━━━ AI Retrieval Server ━━━
+# ==============================================================================================
+#
+# PURPOSE
+# ==============================================================================================
+# Starts ai_serve.js, the retrieval-only HTTP bridge that lets Open-WebUI — which runs in its
+# own container and cannot see Varaverk's filesystem — search the AI index for grounding
+# context.
+#
+# ==============================================================================================
+#
+# OPERATIONAL MODEL
+# ==============================================================================================
+# Mirrors Arrs_Stack/start_webhook_listener.sh, because it solves the same problem: a caller
+# outside Unraid's nginx needs to reach Varaverk, and nginx applies auth_request to everything
+# it serves. Run from ARRAY_START_SCRIPTS and exec's node, so the process this script becomes
+# is the server itself — no PID file to go stale.
+#
+# Retrieval only. Generation stays in ai_query.sh; Open-WebUI has its own model loaded and
+# generates from the chunks this returns.
+#
+# ==============================================================================================
+#
+# DESIGN PRINCIPLES
+# ==============================================================================================
+# Every precondition is checked before exec, not after. A failure at array start should name
+# its cause in the log rather than surface as an exec error once the setup has already run.
+#
+# Disabled is a first-class state. AI_HTTP_PORT=0 or AI_ENABLED=false exits 0 without warning,
+# so a host that does not want the bridge is not a host reporting a failed start script.
+#
+# The index is not built here. This serves an index; ai_index.sh creates one. A server that
+# silently indexed on boot would turn a restart into an unbounded embedding run.
+#
+# ==============================================================================================
+#
+# OPERATIONAL SAFEGUARDS
+# ==============================================================================================
+#
+# Root Required
+#   Writes the generated secret back into master.conf and logs under /var/log/varaverk.
+#
+# Refuses to Serve a Missing Index
+#   The endpoint would answer every query with a retrieval error and Open-WebUI would render
+#   it as "nothing found" — indistinguishable from a corpus that genuinely lacks the answer.
+#   Exits with a message naming ai_index.sh instead.
+#
+# Secret Generated Once, and Verified Persisted
+#   An unset AI_HTTP_SECRET is generated with openssl and written to master.conf. If the
+#   write-back cannot be confirmed the start is failed: a secret that exists only in this
+#   process changes on every restart, silently breaking the tool registered in Open-WebUI.
+#   Same failure mode, and same guard, as the webhook listener's secret.
+#
+# Single Instance
+#   acquire_lock "continuous" — an array stop/start without a reboot leaves the old node
+#   process holding the port, and a second bind would fail with EADDRINUSE and log a false
+#   failure against array_started.sh.
+#
+# Binds Only Where It Must
+#   Defaults to 0.0.0.0 because the caller is a container on the docker bridge and cannot
+#   reach a loopback-bound socket. That is the reason the secret exists, and why the served
+#   surface is a single read-only verb over documentation already in git.
+#
+# ==============================================================================================
+#
+# CONFIGURATION
+# ==============================================================================================
+#   AI_ENABLED              master.conf — false exits without starting
+#   AI_HTTP_PORT            master.conf — 0 disables the bridge
+#   AI_HTTP_SECRET          master.conf — auto-generated on first start if empty
+#   AI_HTTP_BIND            master.conf — bind address, default 0.0.0.0
+#   AI_INDEX_DB             master.conf — index served
+#   <HOST>_OLLAMA_URL       host conf   — embedding endpoint
+#   <HOST>_OLLAMA_EMBED_MODEL host conf — embedding model
+#
+# ==============================================================================================
+#
+# RUNTIME MODES
+# ==============================================================================================
+#   start_ai_server.sh      exec's the server in the foreground; run from ARRAY_START_SCRIPTS
+#
+# ==============================================================================================
+
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+source "$ECOSYSTEM_ROOT/load_config.sh"
+
+if [[ "$EUID" -ne 0 ]]; then
+    error "Must be run as root"
+    exit 1
+fi
+
+if [[ "${AI_ENABLED:-false}" != "true" ]]; then
+    echo "[ai_serve] AI_ENABLED is not true — retrieval bridge disabled"
+    exit 0
+fi
+
+[[ "${AI_HTTP_PORT:-0}" -eq 0 ]] && {
+    echo "[ai_serve] AI_HTTP_PORT=0 — retrieval bridge disabled"
+    exit 0
+}
+
+if ! command -v node >/dev/null 2>&1; then
+    error "node not found — required to run ai_serve.js"
+    notify "AI retrieval bridge failed to start on $(hostname) — node not installed" \
+        "AI Retrieval" "warning"
+    exit 1
+fi
+
+detect_hosts
+
+_url_var="${MY_ID}_OLLAMA_URL"
+_emb_var="${MY_ID}_OLLAMA_EMBED_MODEL"
+OLLAMA_URL="${!_url_var:-}"
+EMBED_MODEL="${!_emb_var:-nomic-embed-text}"
+DB="${AI_INDEX_DB:-${DATA_DIR}/ai_index.db}"
+
+if [[ -z "$OLLAMA_URL" ]]; then
+    error "${MY_ID}_OLLAMA_URL is empty — cannot embed queries"
+    exit 1
+fi
+
+# Serving an absent index answers every question with a retrieval error, which Open-WebUI
+# renders as "nothing found" — the same thing an empty corpus looks like.
+if [[ ! -f "$DB" ]]; then
+    error "No index at $DB — run AI/ai_index.sh first"
+    notify "AI retrieval bridge not started on $(hostname) — index missing" \
+        "AI Retrieval" "warning"
+    exit 1
+fi
+
+acquire_lock "continuous"
+
+if [[ -z "${AI_HTTP_SECRET:-}" ]]; then
+    if ! command -v openssl >/dev/null 2>&1; then
+        error "openssl not found — cannot generate AI_HTTP_SECRET"
+        exit 1
+    fi
+
+    GENERATED=$(openssl rand -hex 32)
+    MASTER_CONF="$ECOSYSTEM_ROOT/Configurations/master.conf"
+    sed -i "s/AI_HTTP_SECRET=\"\"/AI_HTTP_SECRET=\"$GENERATED\"/" "$MASTER_CONF"
+    AI_HTTP_SECRET="$GENERATED"
+
+    # A secret held only in this process would differ on the next start, silently breaking
+    # the tool already registered in Open-WebUI. Same guard as the webhook listener.
+    if ! grep -q "AI_HTTP_SECRET=\"$GENERATED\"" "$MASTER_CONF" 2>/dev/null; then
+        error "Generated AI_HTTP_SECRET but could not persist it to $MASTER_CONF"
+        error "Set AI_HTTP_SECRET manually — a non-persisted secret changes on every restart"
+        notify "AI retrieval secret not persisted on $(hostname)" "AI Retrieval" "warning"
+        exit 1
+    fi
+
+    echo "[ai_serve] Generated AI_HTTP_SECRET — add it to the Open-WebUI tool"
+fi
+
+mkdir -p /var/log/varaverk
+
+exec node "$ECOSYSTEM_ROOT/AI/ai_serve.js" \
+    "$AI_HTTP_PORT" "$AI_HTTP_SECRET" "${AI_HTTP_BIND:-0.0.0.0}" \
+    "$DB" "$OLLAMA_URL" "$EMBED_MODEL" \
+    >> /var/log/varaverk/ai_serve.log 2>&1

Deployment/master.conf.template

@@ -302,6 +302,7 @@
         "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
         "Arrs_Stack/arr_cache_prefill.sh"               # warm Lidarr/Sonarr/Radarr tracked-data caches before anything reads them cold
         "Arrs_Stack/start_webhook_listener.sh"             # arr upgrade webhook listener — continuous
+        "AI/start_ai_server.sh"                       # retrieval bridge for Open-WebUI — continuous, self-disables if AI_ENABLED=false
         "Fallback/fallback.sh"                        # mutual failover — continuous
     )
 
@@ -1625,6 +1626,19 @@
     AI_SEARCH_K=8                   # chunks retrieved per query
     AI_SEARCH_PER_FILE=3            # cap per file so one document cannot fill the context
 
+# ━━━ AI Retrieval Bridge ━━━
+# AI/ai_serve.js — a retrieval-only HTTP endpoint for clients that cannot see this filesystem.
+# Open-WebUI runs in its own container with no WebGUI session, and nginx applies auth_request to
+# everything it serves, so a container cannot reach the plugin's PHP API. Same reasoning, and
+# same shape, as WEBHOOK_PORT / WEBHOOK_SECRET above.
+#
+# It serves chunks, never answers — the caller already has a model. Read-only over documentation
+# that is already in git; the secret is the only gate, so treat the port as public and keep the
+# surface at one verb. Port 0 disables it.
+    AI_HTTP_PORT=7822
+    AI_HTTP_SECRET=""               # auto-generated on first start if empty
+    AI_HTTP_BIND="0.0.0.0"          # must be reachable from the docker bridge, not just loopback
+
 # ━━━ AI Feature Toggles ━━━
 # Tier 1 is narration — it cannot change a decision. Tier 2 adds context to a decision a script
 # already made. Tier 3 assists a human. Enable in that order, and give each one weeks.