diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index b0c3a3e..ef009af 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -1764,6 +1764,14 @@ AI_ASSIST_CLEANUP=false # tier 2 — orphan and stuck-import triage AI_ASSIST_ONBOARD=false # tier 3 — onboarding / settings assistance +# ━━━ AI Repair Findings ━━━ +# Misconfigurations found in this installation, as opposed to defects in Varaverk — those go to +# ai_bugs. A finding is open until the configuration is right, and closes itself when the probe +# that proved the fault starts passing. Closed ones are kept for a while, because "this happened +# before and here is what fixed it" is worth more than the disk. Open findings are never pruned: +# an unresolved problem does not stop mattering because it is old. + AI_FINDING_RETAIN_DAYS=90 + # ━━━ AI Conf Write Access ━━━ # Separate switch from AI_ENABLED, off by default, and an explicit key whitelist. Never paths, # never credentials, never a container name. An empty whitelist means no writes regardless of diff --git a/Plugin/unraid/include/ai_repair.php b/Plugin/unraid/include/ai_repair.php new file mode 100644 index 0000000..061588d --- /dev/null +++ b/Plugin/unraid/include/ai_repair.php @@ -0,0 +1,504 @@ + 'a configured address or port refused, timed out, or did not resolve', + 'auth_rejected' => 'the endpoint answered, and rejected the credential', + 'unknown_target' => 'a conf entry names a container or share that does not exist here', + 'missing_value' => 'a conf key required by the job that ran is empty', +]; + +// How a finding ended, when it ends. +const VV_AI_FINDING_STATES = [ + 'open' => 'seen, not yet acted on', + 'needs_operator' => 'cannot be repaired here — the value is not derivable from this host', + 'acknowledged' => 'the operator knows, and it stays quiet until the state it was acked at changes', + 'fixed' => 'a proven value was written to conf', + 'resolved' => 'the probe now passes; whatever was wrong is no longer wrong', + 'dismissed' => 'the operator says this is not a problem, permanently', +]; + +// ── Why acknowledged is not dismissed ──────────────────────────────────────────────────────── +// "I know critical rsync is off, stop telling me" and "this is never a problem" are different +// instructions, and collapsing them loses the half that matters. An acknowledgement is scoped to +// the state it was given in: CRITICAL_RSYNC_ENABLED being false is a deliberate choice today and +// a stale note the moment it goes true again. +// +// So an ack records what the key read when it was given, and expires when that changes. The +// finding comes back on its own, without the operator having to remember to look — which is the +// difference between a note and a silence. + +// ── Toggles are the operator's, always ─────────────────────────────────────────────────────── +// A repair may never enable or disable anything on its own. Not because it would get the value +// wrong — a boolean has only two — but because the value is not a fact to be discovered. Whether +// critical rsync should be on is a decision about intent, and a probe cannot prove intent the +// way it can prove that a port answers. +// +// The Fix action still writes it when the operator asks for it. What is forbidden is the +// unattended path choosing for them. +function vv_ai_conf_is_toggle(string $key): bool { + $v = strtolower(trim((string)(vv_conf_vars()[$key] ?? ''))); + return $v === 'true' || $v === 'false'; +} + +// May the sweep write this without being asked? Two conditions, both required: a probe actually +// answered on the proposed value, and the key is not a toggle. +function vv_ai_finding_may_autofix(array $f): bool { + if (empty($f['proven'])) return false; + if (($f['proposed'] ?? null) === null) return false; + if (vv_ai_conf_is_toggle((string)($f['conf_key'] ?? ''))) return false; + return true; +} + +// ── Severity is derived, never supplied ────────────────────────────────────────────────────── +// Same ladder run_job.sh records runs against — ok / warn / error — so a finding and the run it +// came from cannot describe the same event at two different volumes. +// +// The rule that matters: a finding whose key is a toggle can never be an error. Something not +// happening because it was switched off is the switch working. That is true whether the switch +// was flipped deliberately last month or by accident this morning, and the store cannot tell +// those apart — so it reports the fact and lets the operator supply the intent. +// +// Everything else takes its level from what the fault costs. A credential the endpoint rejected +// stops that integration dead; an address that does not answer might be a host still booting. +function vv_ai_finding_severity(array $f): string { + $key = (string)($f['conf_key'] ?? ''); + + // Deliberate-state findings never escalate, whatever their kind. + if (vv_ai_conf_is_toggle($key)) return 'warn'; + + return match ($f['kind'] ?? '') { + 'auth_rejected' => 'error', // answered and refused — nothing gets through until fixed + 'missing_value' => 'error', // configured to use something that was never supplied + 'unreachable' => 'warn', // may be transient; the strike system is what escalates it + 'unknown_target' => 'warn', + default => 'warn', + }; +} + +function vv_ai_findings_dir(): string { + $d = AI_DATA_DIR . '/ai_findings'; + if (!is_dir($d)) @mkdir($d, 0755, true); + return $d; +} + +function vv_ai_finding_retain_days(): int { + $n = (int)(vv_conf_vars()['AI_FINDING_RETAIN_DAYS'] ?? 90); + return max(1, $n); +} + +// kind + subject + conf key. Deliberately not the message: the same wrong port produces slightly +// different log text as the software around it changes, and that must not mint a second record. +function vv_ai_finding_id(string $kind, string $subject, string $confKey): string { + return substr(sha1(strtolower($kind . '|' . $subject . '|' . $confKey)), 0, 12); +} + +function vv_ai_finding_path(string $id): ?string { + if (!preg_match('/^[0-9a-f]{12}$/', $id)) return null; + return vv_ai_findings_dir() . '/' . $id . '.json'; +} + +function vv_ai_finding_get(string $id): ?array { + $p = vv_ai_finding_path($id); + if ($p === null || !is_file($p)) return null; + $r = json_decode((string)@file_get_contents($p), true); + return is_array($r) ? $r : null; +} + +// Files a finding, or increments the one already describing this fault. +// +// $f expects: kind, subject, conf_key, conf_file, observed, evidence, source_log +// and optionally: proposed, proven, state, note +function vv_ai_finding_write(array $f): array { + $kind = (string)($f['kind'] ?? ''); + $subject = trim((string)($f['subject'] ?? '')); + $confKey = trim((string)($f['conf_key'] ?? '')); + $evidence = trim((string)($f['evidence'] ?? '')); + + if (!isset(VV_AI_FINDING_KINDS[$kind])) return ['ok' => false, 'error' => 'unknown kind']; + if ($subject === '' || $confKey === '') return ['ok' => false, 'error' => 'subject and conf_key required']; + if ($evidence === '') return ['ok' => false, 'error' => 'evidence required']; + // The key has to be a real shell identifier for the same reason the conf writer insists on + // it: a finding is a proposal to edit that key, and a malformed one can never be actioned. + if (!vv_conf_key_valid($confKey)) return ['ok' => false, 'error' => 'malformed conf key']; + + $state = (string)($f['state'] ?? 'open'); + if (!isset(VV_AI_FINDING_STATES[$state])) $state = 'open'; + + $now = time(); + $id = vv_ai_finding_id($kind, $subject, $confKey); + $rec = [ + 'id' => $id, + 'kind' => $kind, + 'subject' => mb_substr($subject, 0, 120), + 'conf_key' => $confKey, + 'conf_file' => (string)($f['conf_file'] ?? 'master.conf'), + // Secrets never enter this store. A finding about a rejected API key is about the key + // being wrong, and the wrong value is of no use to anyone reading the record later. + 'observed' => vv_conf_key_is_secret($confKey) ? '' + : mb_substr((string)($f['observed'] ?? ''), 0, 300), + 'proposed' => isset($f['proposed']) && !vv_conf_key_is_secret($confKey) + ? mb_substr((string)$f['proposed'], 0, 300) : null, + 'proven' => (bool)($f['proven'] ?? false), + 'state' => $state, + 'evidence' => mb_substr(vv_ai_redact($evidence), 0, 1200), + 'source_log' => mb_substr((string)($f['source_log'] ?? ''), 0, 200), + 'note' => mb_substr((string)($f['note'] ?? ''), 0, 1000), + // Recomputed on every sighting rather than stored once: a key that becomes a toggle, or + // a toggle that is replaced by a real value, changes what this finding means. + 'severity' => vv_ai_finding_severity(['kind' => $kind, 'conf_key' => $confKey]), + 'host' => vv_detect_host(), + 'first' => $now, + 'last' => $now, + 'seen' => 1, + 'closed_at' => null, + // What the key read when the operator acknowledged it. Null unless acked; the ack + // expires the moment the live value stops matching this. + 'ack_value' => null, + ]; + + $p = vv_ai_finding_path($id); + if ($p === null) return ['ok' => false, 'error' => 'bad id']; + + if (is_file($p)) { + $old = json_decode((string)@file_get_contents($p), true); + if (is_array($old)) { + $rec['first'] = $old['first'] ?? $now; + $rec['seen'] = (int)($old['seen'] ?? 0) + 1; + // A dismissed finding stays dismissed however many times the log repeats it — + // otherwise "this is fine, stop telling me" lasts exactly one cycle. A fixed one + // reopens, because seeing the fault again after a repair means the repair did not + // hold, which is the single most important thing this store can tell anyone. + if (($old['state'] ?? '') === 'dismissed') { + $rec['state'] = 'dismissed'; + $rec['closed_at'] = $old['closed_at'] ?? null; + } + // An acknowledgement holds only while the thing acknowledged is still true. Compare + // the live value against what it read when the ack was given: unchanged means stay + // quiet, changed means the note is stale and the finding comes back by itself. + if (($old['state'] ?? '') === 'acknowledged') { + $ackedAt = (string)($old['ack_value'] ?? ''); + if ($ackedAt === (string)(vv_conf_vars()[$confKey] ?? '')) { + $rec['state'] = 'acknowledged'; + $rec['ack_value'] = $ackedAt; + $rec['closed_at'] = $old['closed_at'] ?? null; + } + // Otherwise $rec keeps the state this sighting computed — it has reopened. + } + // Preserve an operator's note over a generated one. + if ($rec['note'] === '' && !empty($old['note'])) $rec['note'] = $old['note']; + } + } + + if (@file_put_contents($p, json_encode($rec, JSON_PRETTY_PRINT)) === false) { + return ['ok' => false, 'error' => 'write failed']; + } + return ['ok' => true, 'id' => $id, 'seen' => $rec['seen'], 'state' => $rec['state']]; +} + +// $states filters; empty means everything. Newest activity first, because a finding seen in the +// last cycle matters more than one that has been sitting fixed for a month. +function vv_ai_findings_list(array $states = ['open', 'needs_operator']): array { + $out = []; + foreach ((array)@glob(vv_ai_findings_dir() . '/*.json') as $file) { + $r = json_decode((string)@file_get_contents($file), true); + if (!is_array($r)) continue; + if ($states && !in_array($r['state'] ?? 'open', $states, true)) continue; + $out[] = $r; + } + usort($out, fn($a, $b) => ($b['last'] ?? 0) <=> ($a['last'] ?? 0)); + return $out; +} + +function vv_ai_finding_set_state(string $id, string $state, string $note = ''): bool { + if (!isset(VV_AI_FINDING_STATES[$state])) return false; + $r = vv_ai_finding_get($id); + if ($r === null) return false; + + $r['state'] = $state; + $r['closed_at'] = in_array($state, ['open', 'needs_operator'], true) ? null : time(); + if ($note !== '') $r['note'] = mb_substr(vv_ai_redact($note), 0, 1000); + + $p = vv_ai_finding_path($id); + return $p !== null && @file_put_contents($p, json_encode($r, JSON_PRETTY_PRINT)) !== false; +} + +function vv_ai_finding_close(string $id, string $note = ''): bool { + return vv_ai_finding_set_state($id, 'resolved', $note); +} + +function vv_ai_finding_dismiss(string $id, string $note = ''): bool { + return vv_ai_finding_set_state($id, 'dismissed', $note); +} + +// "I know about this — leave it, and tell me if it changes." +// +// Stamps the key's current value onto the record. Every later sighting compares against that +// stamp, so the acknowledgement covers this state and not the key forever. Acking that critical +// rsync is off says nothing about critical rsync being on. +function vv_ai_finding_ack(string $id, string $note = ''): bool { + $r = vv_ai_finding_get($id); + if ($r === null) return false; + + $r['state'] = 'acknowledged'; + $r['ack_value'] = (string)(vv_conf_vars()[$r['conf_key'] ?? ''] ?? ''); + $r['closed_at'] = time(); + if ($note !== '') $r['note'] = mb_substr(vv_ai_redact($note), 0, 1000); + + $p = vv_ai_finding_path($id); + return $p !== null && @file_put_contents($p, json_encode($r, JSON_PRETTY_PRINT)) !== false; +} + +// What the operator can do about a finding, and what each choice means. Returned rather than +// hardcoded in the UI so the chat and the page cannot offer different options for the same row. +// +// Fix appears for anything with a proposed value, toggle or not — the prohibition is on the +// sweep choosing, never on the operator choosing. Everything carries ack and cancel, because +// "I know" and "not now" are always valid answers to being told something. +function vv_ai_finding_actions(array $f): array { + $actions = []; + + if (($f['proposed'] ?? null) !== null) { + $actions['fix'] = vv_ai_conf_is_toggle((string)($f['conf_key'] ?? '')) + ? 'Set ' . $f['conf_key'] . ' — a toggle, so this only ever happens because you asked' + : 'Write the proven value to ' . $f['conf_key']; + } + + $actions['ack'] = 'Known and intended. Stays quiet until ' . ($f['conf_key'] ?? 'it') . ' changes'; + $actions['cancel'] = 'Leave it alone for now'; + + return $actions; +} + +// Closed findings are kept for a while because "this happened before and here is what fixed it" +// is worth more than the disk it costs. Open ones are never pruned — an unresolved problem does +// not stop mattering because it is old. +function vv_ai_findings_prune(): int { + $cutoff = time() - (vv_ai_finding_retain_days() * 86400); + $n = 0; + foreach ((array)@glob(vv_ai_findings_dir() . '/*.json') as $file) { + $r = json_decode((string)@file_get_contents($file), true); + if (!is_array($r)) continue; + if (in_array($r['state'] ?? 'open', ['open', 'needs_operator'], true)) continue; + if ((int)($r['closed_at'] ?? 0) > $cutoff) continue; + if (@unlink($file)) $n++; + } + return $n; +} + +// What the assistant should raise when a page loads: things that need the operator, newest +// first. Repaired findings are deliberately not here — a fix that worked is a log entry, not a +// conversation, and opening every session with a list of things that already went right is how +// an operator learns to close the panel without reading it. +function vv_ai_findings_for_chat(int $limit = 3): array { + return array_slice(vv_ai_findings_list(['needs_operator']), 0, max(1, $limit)); +} + +// ── Resolving a log line back to a conf key ────────────────────────────────────────────────── +// By value wherever possible, by name only as a fallback. +// +// A log line usually contains the thing that failed — the URL that did not answer. That value +// came from a conf key, so searching the conf for which key holds it is an exact lookup with a +// definite answer. Guessing the key from the container's name is inference, and the failure mode +// is silent: HOST1_EMBY_URL and HOST1_EMBY_EXTERNAL_URL are both plausible for "Emby" and only +// one of them is the value that just failed. +// +// Same principle as resolve_tailscale_ip() refusing similarity matching for host identity: an +// exact match or an honest nothing. + +// Every conf key whose value the given text starts with, longest first. Prefix rather than +// equality because a log reports the URL it actually called — the conf value plus an endpoint +// path — and the longest match is the most specific key that could have produced it. +function vv_ai_conf_keys_for_value(string $value): array { + $value = trim($value); + if ($value === '' || strlen($value) < 6) return []; + + $hits = []; + foreach (vv_conf_vars() as $k => $v) { + $v = trim((string)$v); + if ($v === '' || strlen($v) < 6) continue; + if ($v === $value || str_starts_with($value, $v)) $hits[$k] = strlen($v); + } + arsort($hits); + return array_keys($hits); +} + +// __, built literally and then checked for existence. Nothing is +// inferred: either the conf holds a key by exactly that name or this returns null. +function vv_ai_conf_key_for_subject(string $subject, string $suffix): ?string { + $norm = strtoupper(preg_replace('/[^A-Za-z0-9]+/', '_', trim($subject))); + if ($norm === '') return null; + + $key = strtoupper(vv_detect_host()) . '_' . $norm . '_' . strtoupper($suffix); + return array_key_exists($key, vv_conf_vars()) ? $key : null; +} + +// Which conf file a key lives in. A finding has to name the file it would be edited in, and +// host keys are not in master.conf. +function vv_ai_conf_file_for_key(string $key): string { + foreach (vv_get_conf_files() as $f) { + if (preg_match('/^\s*' . preg_quote($key, '/') . '\s*=/m', vv_read_conf_raw($f))) return $f; + } + return 'master.conf'; +} + +// ── Deterministic triage ───────────────────────────────────────────────────────────────────── +// Patterns are written against log formats that exist in this repo today, taken from the emit +// sites rather than imagined. A pattern that stops matching because its log line was reworded +// produces no finding, which is the safe direction — the alternative is a pattern loose enough +// to match anything, which fills the store with rows nobody can act on. +// +// No model runs here. "Connection refused" is not a judgement call, and a 14B model invoked +// after every cron job to notice it would be slower, costlier and less reliable than a regex. +// The model's job starts where these stop: explaining a finding, and talking the operator +// through the ones that cannot be repaired automatically. +const VV_AI_TRIAGE_PATTERNS = [ + // docker_watchdog HTTP check — the richest signal, carrying both subject and failing URL. + [ + 're' => '/^(?P\S+) — not responding at (?P\S+) \(strike/u', + 'kind' => 'unreachable', + 'suffix' => 'URL', + ], + // docker_watchdog API check, 401/403. The endpoint answered, so the address is right and + // the credential is not. + [ + 're' => '/^(?P\S+) — API check skipped \(HTTP (?:401|403)/u', + 'kind' => 'auth_rejected', + 'suffix' => 'API_KEY', + ], + // Configured for an API check with nothing to authenticate with. + [ + 're' => '/^(?P\S+) — API check skipped \(no key configured\)/u', + 'kind' => 'missing_value', + 'suffix' => 'API_KEY', + ], + [ + 're' => '/^Skipping (?P\S+) — placeholder API key/u', + 'kind' => 'missing_value', + 'suffix' => 'API_KEY', + ], + [ + 're' => '/^(?P\S+) (?:—\s*)?API unreachable/u', + 'kind' => 'unreachable', + 'suffix' => 'URL', + ], +]; + +// One log line in, at most one finding candidate out. Returns null for everything else, which is +// almost every line. +function vv_ai_triage_line(string $line): ?array { + // Strip the timestamp and level decoration the logger adds, so patterns can anchor on ^. + $body = preg_replace('/^\S+\s+\S+\s+(?:[^\[]*\[[A-Z]+\]\s*)?/u', '', rtrim($line)); + $body = trim((string)$body); + if ($body === '') return null; + + foreach (VV_AI_TRIAGE_PATTERNS as $p) { + if (!preg_match($p['re'], $body, $m)) continue; + + $subject = trim($m['subject'] ?? ''); + $observed = trim($m['observed'] ?? ''); + if ($subject === '') continue; + + // Value first, name second. Only one candidate key is accepted — two keys holding the + // same value means the log cannot say which one produced it, and picking either is the + // guess this whole approach exists to avoid. + $key = null; + if ($observed !== '') { + $byValue = vv_ai_conf_keys_for_value($observed); + if (count($byValue) === 1) $key = $byValue[0]; + } + if ($key === null) $key = vv_ai_conf_key_for_subject($subject, $p['suffix']); + if ($key === null) continue; // nothing actionable — no row + + return [ + 'kind' => $p['kind'], + 'subject' => $subject, + 'conf_key' => $key, + 'conf_file' => vv_ai_conf_file_for_key($key), + 'observed' => $observed !== '' ? $observed : (string)(vv_conf_vars()[$key] ?? ''), + 'evidence' => $body, + ]; + } + return null; +} + +// A whole log tail in, one candidate per distinct fault out. A job that retried twelve times +// produces twelve identical lines, and the store's dedupe would collapse them anyway — doing it +// here keeps the sweep from writing the same file twelve times in a row. +function vv_ai_triage_log(array $lines, string $sourceLog = ''): array { + $found = []; + foreach ($lines as $line) { + $c = vv_ai_triage_line((string)$line); + if ($c === null) continue; + $c['source_log'] = $sourceLog; + $found[vv_ai_finding_id($c['kind'], $c['subject'], $c['conf_key'])] = $c; + } + return array_values($found); +}