diff --git a/AI/README-AI.md b/AI/README-AI.md index f06cfc0..048f46c 100644 --- a/AI/README-AI.md +++ b/AI/README-AI.md @@ -216,17 +216,12 @@ built before any of that is enabled. ## ━━━ SCHEDULING ━━━ -Not scheduled by default. When you want the index to track the repo automatically, the natural -home is after a successful pull — the only moment the corpus actually changes: +The index tracks the repo automatically, from the only moment the corpus actually changes — a +successful pull. `git_pull_execute.sh` re-indexes behind three gates: the pull succeeded and +changed tracked files, `AI_INDEX_ON_PULL=true`, and `AI_ENABLED=true`. It is never fatal — a +failed index leaves the previous one in place and the pull still reports success. -```bash -# master.conf → DAILY_MAINTENANCE_SCRIPTS, after git_pull_execute.sh -"AI/ai_index.sh" -``` - -`AI_INDEX_ON_PULL` exists in `master.conf` as the intended gate for wiring this into -`git_pull_execute.sh` directly. It is not yet consumed by anything — the variable is reserved, -not live. +No cron entry and no `DAILY_MAINTENANCE_SCRIPTS` line are needed; the daily pull carries it. An incremental run on an unchanged repo is ~70 ms, so a daily entry costs effectively nothing and a pull that changed twelve files costs a few seconds. diff --git a/AI/ai_serve.js b/AI/ai_serve.js deleted file mode 100755 index dd99ee7..0000000 --- a/AI/ai_serve.js +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env node -// ═══════════════════════════════════════════════════════════════════════════════════════════════ -// PURPOSE -// Retrieval-only HTTP bridge over the AI index. Lets a client that cannot see Varaverk's -// filesystem — Open-WebUI in its own container — search the corpus and receive grounding -// chunks as JSON. -// -// OPERATIONAL MODEL -// Started by start_ai_server.sh, which is the only thing that knows the port and secret. -// Lives outside Unraid's nginx exactly as webhook_listener.js does, because the caller is a -// container with no WebGUI session and nginx applies auth_request to everything it serves. -// The shared secret in the query string is therefore the only gate. -// -// Returns chunks, never answers. Open-WebUI already has a model loaded and is going to -// generate from whatever this returns; generating here as well would double the latency and -// burn a second model load for a result the caller discards. ai_query.sh keeps the -// generation path for CLI use — this endpoint deliberately does not. -// -// Calls search() in-process rather than shelling out to ai_query.sh. A shell round trip per -// request would fork bash, re-source three conf files and re-open the index every time, for -// a call that is otherwise a single embed plus a vector scan. -// -// DESIGN PRINCIPLES -// One endpoint, one verb. -// GET /search. No index management, no conf access, no generation. The surface an -// unauthenticated port exposes should be the smallest thing that does the job. -// -// The caller chooses breadth, within bounds. -// k, kind and section are accepted because the client knows what kind of question it is -// asking. All are clamped or validated here — the caller is trusted to express intent, -// not to be correct. -// -// Failures answer in the same shape as successes. -// Every response is JSON with an ok flag. A tool calling this has no way to render an -// HTML error page, and a caller that cannot parse the failure reports nothing at all. -// -// OPERATIONAL SAFEGUARDS -// The secret is compared at fixed length before anything else runs. -// No index is opened and no embedding is requested until the key matches, so an -// unauthenticated caller cannot make this process do work. Compared with -// timingSafeEqual on equal-length buffers — a plain !== leaks length and position -// through response timing on a port that is, by design, reachable without a session. -// -// Read-only by construction. -// The index is opened by search() for reading and nothing here writes to it, to conf, -// or to the filesystem. The worst a valid key can do is read documentation that is -// already in a git repository. -// -// Bounded work per request. -// k is clamped to 1..25 and the query to 2000 characters, so no single call can pull the -// whole index into memory or embed an unbounded string. -// -// Bad input is refused, not guessed. -// kind is checked against the five real values; an unknown one is rejected rather than -// passed to SQL where it would match zero rows and read as "the index has no answer" — -// the most misleading failure this system can produce. -// -// Binds where it is told, and says so. -// The bind address comes from the launcher. It is reachable from the docker bridge by -// necessity; that is why the secret exists and why the surface is one read-only verb. -// -// CONFIGURATION -// argv: -// All supplied by start_ai_server.sh from AI_HTTP_PORT, AI_HTTP_SECRET, AI_INDEX_DB and -// _OLLAMA_URL / _OLLAMA_EMBED_MODEL. -// -// RUNTIME MODES -// GET /search?key=…&q=…[&k=8][&kind=readme][§ion=PURPOSE] -// GET /health?key=… liveness only — no index access -// ═══════════════════════════════════════════════════════════════════════════════════════════════ - -'use strict'; - -const http = require('http'); -const crypto = require('crypto'); -const path = require('path'); - -const [, , port, secret, bind, db, ollamaUrl, embedModel] = process.argv; - -if (!port || !secret || !bind || !db || !ollamaUrl || !embedModel) { - process.stderr.write('Usage: ai_serve.js \n'); - process.exit(1); -} - -const { search } = require(path.join(__dirname, 'lib', 'search.js')); - -const KINDS = new Set(['header', 'readme', 'manual', 'template', 'doc']); -const MAX_Q = 2000; -const SECRET_B = Buffer.from(secret); - -function send(res, code, obj) { - const body = JSON.stringify(obj); - res.writeHead(code, { - 'Content-Type': 'application/json', - 'Content-Length': Buffer.byteLength(body), - 'Cache-Control': 'no-store', - }); - res.end(body); -} - -// Equal-length compare first: timingSafeEqual throws on a length mismatch, and the length -// itself is not worth leaking through an exception path on an unauthenticated port. -function keyOk(given) { - if (typeof given !== 'string') return false; - const g = Buffer.from(given); - if (g.length !== SECRET_B.length) return false; - return crypto.timingSafeEqual(g, SECRET_B); -} - -const server = http.createServer(async (req, res) => { - let url; - try { url = new URL(req.url, 'http://localhost'); } - catch { return send(res, 400, { ok: false, error: 'Bad request' }); } - - if (!keyOk(url.searchParams.get('key'))) { - return send(res, 403, { ok: false, error: 'Forbidden' }); - } - - if (url.pathname === '/health') { - return send(res, 200, { ok: true, service: 'ai_serve', db }); - } - - if (url.pathname !== '/search') { - return send(res, 404, { ok: false, error: 'Not found' }); - } - - const q = (url.searchParams.get('q') || '').trim(); - if (!q) return send(res, 400, { ok: false, error: 'q is required' }); - if (q.length > MAX_Q) { - return send(res, 400, { ok: false, error: `q exceeds ${MAX_Q} characters` }); - } - - const kind = url.searchParams.get('kind') || null; - if (kind && !KINDS.has(kind)) { - return send(res, 400, { - ok: false, - error: `unknown kind '${kind}' (expected: ${[...KINDS].join(', ')})`, - }); - } - - let k = parseInt(url.searchParams.get('k') || '8', 10); - if (!Number.isFinite(k)) k = 8; - k = Math.min(Math.max(k, 1), 25); - - try { - const r = await search({ - dbPath: db, url: ollamaUrl, embedModel, query: q, - k, perFile: 3, - section: url.searchParams.get('section') || null, - kind, - }); - send(res, 200, { - ok: true, - query: q, - intents: r.intents, - scanned: r.scanned, - results: r.results.map(x => ({ - path: x.path, - section: x.section, - heading: x.heading, - score: Number(x.score.toFixed(4)), - content: x.content, - })), - }); - } catch (e) { - send(res, 500, { ok: false, error: `retrieval failed: ${e.message}` }); - } -}); - -server.on('error', (e) => { - process.stderr.write(`ai_serve: ${e.message}\n`); - process.exit(1); -}); - -server.listen(parseInt(port, 10), bind, () => { - process.stdout.write(`ai_serve listening on ${bind}:${port}\n`); -}); diff --git a/AI/openwebui_tool.py b/AI/openwebui_tool.py deleted file mode 100644 index 2c6d0f3..0000000 --- a/AI/openwebui_tool.py +++ /dev/null @@ -1,129 +0,0 @@ -""" -title: Varaverk Docs -description: Search Varaverk's own documentation and return grounding context. -author: Varaverk -version: 1.0.0 -required_open_webui_version: 0.4.0 -""" - -# ═══════════════════════════════════════════════════════════════════════════════════════════════ -# PURPOSE -# Open-WebUI tool that lets the chat model search Varaverk's documentation index and answer -# from it, instead of from whatever it happens to remember about a private project it has -# never seen. -# -# OPERATIONAL MODEL -# Not installed by any Varaverk script. Open-WebUI stores tools in its own database, and -# writing there directly would mean guessing at its schema, IDs and access control on a live -# app. Paste this into Open-WebUI → Workspace → Tools → +, then set the two Valves. -# -# Calls AI/ai_serve.js over HTTP because Open-WebUI runs in a container that cannot see -# Varaverk's filesystem and has no WebGUI session, so the plugin's PHP API is unreachable to -# it. The bridge returns chunks; the model already loaded in Open-WebUI does the generating. -# -# CONFIGURATION (Valves — set in the Open-WebUI tool editor, not here) -# base_url http://:7822 — AI_HTTP_PORT from master.conf -# secret AI_HTTP_SECRET from master.conf -# -# The secret is a Valve rather than a constant so this file stays committable. Do not paste -# it into the code — this path is git-tracked and pushed to a remote. -# -# OPERATIONAL SAFEGUARDS -# Retrieval only. The bridge exposes one read-only verb over documentation already in git; -# this tool cannot write conf, run a script, or change anything. -# -# Returns "no relevant documentation found" rather than an empty string on a miss, so the -# model states that plainly instead of filling the silence from memory — the failure this -# whole retrieval path exists to prevent. -# -# Every failure is returned as readable text, never raised. An exception inside a tool call -# surfaces to the user as an opaque error; a sentence explaining that the bridge is -# unreachable is something they can act on. -# -# Time-boxed at 30s. A hung retrieval must not hold the chat turn open indefinitely. -# ═══════════════════════════════════════════════════════════════════════════════════════════════ - -import json -import urllib.parse -import urllib.request - -from pydantic import BaseModel, Field - - -class Tools: - class Valves(BaseModel): - base_url: str = Field( - default="http://192.168.50.2:7822", - description="AI retrieval bridge — host LAN IP and AI_HTTP_PORT. Not localhost: " - "Open-WebUI is a container and localhost is itself.", - ) - secret: str = Field( - default="", - description="AI_HTTP_SECRET from Configurations/master.conf", - ) - results: int = Field( - default=8, - description="Chunks to retrieve per query (1-25)", - ) - - def __init__(self): - self.valves = self.Valves() - - def search_varaverk_docs(self, query: str, kind: str = "") -> str: - """ - Search the Varaverk documentation index for passages relevant to a question about - this specific home-media system: its scripts, configuration variables, safeguards, - orchestrators, rsync behaviour, watchdogs, fallback logic or plugin internals. - - Always use this before answering any question about Varaverk. Varaverk is a private - project and is not in your training data; without this tool you do not know what it - is and must not guess. - - :param query: The question or topic to search for, in natural language. - :param kind: Optional filter on where the text comes from. Use "readme" for - definitional or narrative questions such as "what is Varaverk" or "why does this - exist" — otherwise per-script header sections outrank the top-level prose and the - answer will look absent when it is not. Leave empty for specific technical - questions. One of: header, readme, manual, template, doc. - :return: Numbered passages with their source paths, or a message saying nothing matched. - """ - if not self.valves.secret: - return ("The Varaverk docs tool is not configured: its 'secret' Valve is empty. " - "Set it to AI_HTTP_SECRET from Configurations/master.conf.") - - params = { - "key": self.valves.secret, - "q": query, - "k": max(1, min(int(self.valves.results), 25)), - } - if kind: - params["kind"] = kind - - url = f"{self.valves.base_url.rstrip('/')}/search?" + urllib.parse.urlencode(params) - - try: - with urllib.request.urlopen(url, timeout=30) as r: - data = json.load(r) - except Exception as e: - return (f"Could not reach the Varaverk retrieval bridge at " - f"{self.valves.base_url} ({e}). It is started by AI/start_ai_server.sh; " - f"check that AI_ENABLED and AI_HTTP_PORT are set and the index exists.") - - if not data.get("ok"): - return f"Varaverk retrieval failed: {data.get('error', 'unknown error')}" - - results = data.get("results") or [] - if not results: - return (f"No relevant documentation found for '{query}'. Say so plainly rather " - f"than answering from general knowledge — Varaverk is private and is not " - f"in your training data.") - - out = [f"{len(results)} passage(s) from the Varaverk documentation index:", ""] - for i, r in enumerate(results, 1): - label = " › ".join(x for x in (r.get("path"), r.get("section"), r.get("heading")) if x) - out.append(f"[{i}] {label} (score {r.get('score')})") - out.append(r.get("content", "").strip()) - out.append("") - out.append("Answer only from the passages above, and cite them by their [n] markers. " - "If they do not contain the answer, say so and name what is missing.") - return "\n".join(out) diff --git a/AI/start_ai_server.sh b/AI/start_ai_server.sh deleted file mode 100755 index 71f8027..0000000 --- a/AI/start_ai_server.sh +++ /dev/null @@ -1,167 +0,0 @@ -#!/bin/bash -# ============================================================================================== -# ━━━ AI Retrieval Server ━━━ -# ============================================================================================== -# -# PURPOSE -# ============================================================================================== -# Starts ai_serve.js, the retrieval-only HTTP bridge that lets Open-WebUI — which runs in its -# own container and cannot see Varaverk's filesystem — search the AI index for grounding -# context. -# -# ============================================================================================== -# -# OPERATIONAL MODEL -# ============================================================================================== -# Mirrors Arrs_Stack/start_webhook_listener.sh, because it solves the same problem: a caller -# outside Unraid's nginx needs to reach Varaverk, and nginx applies auth_request to everything -# it serves. Run from ARRAY_START_SCRIPTS and exec's node, so the process this script becomes -# is the server itself — no PID file to go stale. -# -# Retrieval only. Generation stays in ai_query.sh; Open-WebUI has its own model loaded and -# generates from the chunks this returns. -# -# ============================================================================================== -# -# DESIGN PRINCIPLES -# ============================================================================================== -# Every precondition is checked before exec, not after. A failure at array start should name -# its cause in the log rather than surface as an exec error once the setup has already run. -# -# Disabled is a first-class state. AI_HTTP_PORT=0 or AI_ENABLED=false exits 0 without warning, -# so a host that does not want the bridge is not a host reporting a failed start script. -# -# The index is not built here. This serves an index; ai_index.sh creates one. A server that -# silently indexed on boot would turn a restart into an unbounded embedding run. -# -# ============================================================================================== -# -# OPERATIONAL SAFEGUARDS -# ============================================================================================== -# -# Root Required -# Writes the generated secret back into master.conf and logs under /var/log/varaverk. -# -# Refuses to Serve a Missing Index -# The endpoint would answer every query with a retrieval error and Open-WebUI would render -# it as "nothing found" — indistinguishable from a corpus that genuinely lacks the answer. -# Exits with a message naming ai_index.sh instead. -# -# Secret Generated Once, and Verified Persisted -# An unset AI_HTTP_SECRET is generated with openssl and written to master.conf. If the -# write-back cannot be confirmed the start is failed: a secret that exists only in this -# process changes on every restart, silently breaking the tool registered in Open-WebUI. -# Same failure mode, and same guard, as the webhook listener's secret. -# -# Single Instance -# acquire_lock "continuous" — an array stop/start without a reboot leaves the old node -# process holding the port, and a second bind would fail with EADDRINUSE and log a false -# failure against array_started.sh. -# -# Binds Only Where It Must -# Defaults to 0.0.0.0 because the caller is a container on the docker bridge and cannot -# reach a loopback-bound socket. That is the reason the secret exists, and why the served -# surface is a single read-only verb over documentation already in git. -# -# ============================================================================================== -# -# CONFIGURATION -# ============================================================================================== -# AI_ENABLED master.conf — false exits without starting -# AI_HTTP_PORT master.conf — 0 disables the bridge -# AI_HTTP_SECRET master.conf — auto-generated on first start if empty -# AI_HTTP_BIND master.conf — bind address, default 0.0.0.0 -# AI_INDEX_DB master.conf — index served -# _OLLAMA_URL host conf — embedding endpoint -# _OLLAMA_EMBED_MODEL host conf — embedding model -# -# ============================================================================================== -# -# RUNTIME MODES -# ============================================================================================== -# start_ai_server.sh exec's the server in the foreground; run from ARRAY_START_SCRIPTS -# -# ============================================================================================== - -set -uo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" - -source "$ECOSYSTEM_ROOT/load_config.sh" - -if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" - exit 1 -fi - -if [[ "${AI_ENABLED:-false}" != "true" ]]; then - echo "[ai_serve] AI_ENABLED is not true — retrieval bridge disabled" - exit 0 -fi - -[[ "${AI_HTTP_PORT:-0}" -eq 0 ]] && { - echo "[ai_serve] AI_HTTP_PORT=0 — retrieval bridge disabled" - exit 0 -} - -if ! command -v node >/dev/null 2>&1; then - error "node not found — required to run ai_serve.js" - notify "AI retrieval bridge failed to start on $(hostname) — node not installed" \ - "AI Retrieval" "warning" - exit 1 -fi - -detect_hosts - -_url_var="${MY_ID}_OLLAMA_URL" -_emb_var="${MY_ID}_OLLAMA_EMBED_MODEL" -OLLAMA_URL="${!_url_var:-}" -EMBED_MODEL="${!_emb_var:-nomic-embed-text}" -DB="${AI_INDEX_DB:-${DATA_DIR}/ai_index.db}" - -if [[ -z "$OLLAMA_URL" ]]; then - error "${MY_ID}_OLLAMA_URL is empty — cannot embed queries" - exit 1 -fi - -# Serving an absent index answers every question with a retrieval error, which Open-WebUI -# renders as "nothing found" — the same thing an empty corpus looks like. -if [[ ! -f "$DB" ]]; then - error "No index at $DB — run AI/ai_index.sh first" - notify "AI retrieval bridge not started on $(hostname) — index missing" \ - "AI Retrieval" "warning" - exit 1 -fi - -acquire_lock "continuous" - -if [[ -z "${AI_HTTP_SECRET:-}" ]]; then - if ! command -v openssl >/dev/null 2>&1; then - error "openssl not found — cannot generate AI_HTTP_SECRET" - exit 1 - fi - - GENERATED=$(openssl rand -hex 32) - MASTER_CONF="$ECOSYSTEM_ROOT/Configurations/master.conf" - sed -i "s/AI_HTTP_SECRET=\"\"/AI_HTTP_SECRET=\"$GENERATED\"/" "$MASTER_CONF" - AI_HTTP_SECRET="$GENERATED" - - # A secret held only in this process would differ on the next start, silently breaking - # the tool already registered in Open-WebUI. Same guard as the webhook listener. - if ! grep -q "AI_HTTP_SECRET=\"$GENERATED\"" "$MASTER_CONF" 2>/dev/null; then - error "Generated AI_HTTP_SECRET but could not persist it to $MASTER_CONF" - error "Set AI_HTTP_SECRET manually — a non-persisted secret changes on every restart" - notify "AI retrieval secret not persisted on $(hostname)" "AI Retrieval" "warning" - exit 1 - fi - - echo "[ai_serve] Generated AI_HTTP_SECRET — add it to the Open-WebUI tool" -fi - -mkdir -p /var/log/varaverk - -exec node "$ECOSYSTEM_ROOT/AI/ai_serve.js" \ - "$AI_HTTP_PORT" "$AI_HTTP_SECRET" "${AI_HTTP_BIND:-0.0.0.0}" \ - "$DB" "$OLLAMA_URL" "$EMBED_MODEL" \ - >> /var/log/varaverk/ai_serve.log 2>&1 diff --git a/Deployment/host.conf.template b/Deployment/host.conf.template index a88e560..8c13a61 100644 --- a/Deployment/host.conf.template +++ b/Deployment/host.conf.template @@ -578,7 +578,6 @@ # ━━━ Ollama ━━━ HOSTN_OLLAMA_URL="" # e.g. http://localhost:11434 — empty if no local Ollama - HOSTN_OLLAMA_CONTAINER="Ollama" # for watchdog / restart lists HOSTN_OLLAMA_GPU_UUID="" # pins Ollama to one card on multi-GPU hosts HOSTN_OLLAMA_MODEL="hf.co/unsloth/Qwen3-14B-GGUF:IQ4_XS" # generation — must fully offload; see README-AI.md HOSTN_OLLAMA_EMBED_MODEL="nomic-embed-text" # embeddings — the generation model cannot embed diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 95517d6..b4193ed 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -302,7 +302,6 @@ "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers "Arrs_Stack/arr_cache_prefill.sh" # warm Lidarr/Sonarr/Radarr tracked-data caches before anything reads them cold "Arrs_Stack/start_webhook_listener.sh" # arr upgrade webhook listener — continuous - "AI/start_ai_server.sh" # retrieval bridge for Open-WebUI — continuous, self-disables if AI_ENABLED=false "Fallback/fallback.sh" # mutual failover — continuous ) @@ -1645,19 +1644,6 @@ AI_MEMORY_FILE="$DATA_DIR/ai_memory.md" AI_MEMORY_MAX_CHARS=4000 # ~1000 tokens — truncated with a notice if exceeded -# ━━━ AI Retrieval Bridge ━━━ -# AI/ai_serve.js — a retrieval-only HTTP endpoint for clients that cannot see this filesystem. -# Open-WebUI runs in its own container with no WebGUI session, and nginx applies auth_request to -# everything it serves, so a container cannot reach the plugin's PHP API. Same reasoning, and -# same shape, as WEBHOOK_PORT / WEBHOOK_SECRET above. -# -# It serves chunks, never answers — the caller already has a model. Read-only over documentation -# that is already in git; the secret is the only gate, so treat the port as public and keep the -# surface at one verb. Port 0 disables it. - AI_HTTP_PORT=7822 - AI_HTTP_SECRET="" # auto-generated on first start if empty - AI_HTTP_BIND="0.0.0.0" # must be reachable from the docker bridge, not just loopback - # ━━━ AI Feature Toggles ━━━ # Tier 1 is narration — it cannot change a decision. Tier 2 adds context to a decision a script # already made. Tier 3 assists a human. Enable in that order, and give each one weeks.