'a configured address or port refused, timed out, or did not resolve', 'auth_rejected' => 'the endpoint answered, and rejected the credential', 'unknown_target' => 'a conf entry names a container or share that does not exist here', 'missing_value' => 'a conf key required by the job that ran is empty', ]; // How a finding ended, when it ends. const VV_AI_FINDING_STATES = [ 'open' => 'seen, not yet acted on', 'needs_operator' => 'cannot be repaired here — the value is not derivable from this host', 'acknowledged' => 'the operator knows, and it stays quiet until the state it was acked at changes', 'fixed' => 'a proven value was written to conf', 'resolved' => 'the probe now passes; whatever was wrong is no longer wrong', 'dismissed' => 'the operator says this is not a problem, permanently', ]; // ── Why acknowledged is not dismissed ──────────────────────────────────────────────────────── // "I know critical rsync is off, stop telling me" and "this is never a problem" are different // instructions, and collapsing them loses the half that matters. An acknowledgement is scoped to // the state it was given in: CRITICAL_RSYNC_ENABLED being false is a deliberate choice today and // a stale note the moment it goes true again. // // So an ack records what the key read when it was given, and expires when that changes. The // finding comes back on its own, without the operator having to remember to look — which is the // difference between a note and a silence. // ── Toggles are the operator's, always ─────────────────────────────────────────────────────── // A repair may never enable or disable anything on its own. Not because it would get the value // wrong — a boolean has only two — but because the value is not a fact to be discovered. Whether // critical rsync should be on is a decision about intent, and a probe cannot prove intent the // way it can prove that a port answers. // // The Fix action still writes it when the operator asks for it. What is forbidden is the // unattended path choosing for them. function vv_ai_conf_is_toggle(string $key): bool { $v = strtolower(trim((string)(vv_conf_vars()[$key] ?? ''))); return $v === 'true' || $v === 'false'; } // May the sweep write this without being asked? Two conditions, both required: a probe actually // answered on the proposed value, and the key is not a toggle. function vv_ai_finding_may_autofix(array $f): bool { if (!vv_ai_repair_autofix_enabled()) return false; if (empty($f['proven'])) return false; if (($f['proposed'] ?? null) === null) return false; if (vv_ai_conf_is_toggle((string)($f['conf_key'] ?? ''))) return false; return true; } // ── Severity is derived, never supplied ────────────────────────────────────────────────────── // Same ladder run_job.sh records runs against — ok / warn / error — so a finding and the run it // came from cannot describe the same event at two different volumes. // // The rule that matters: a finding whose key is a toggle can never be an error. Something not // happening because it was switched off is the switch working. That is true whether the switch // was flipped deliberately last month or by accident this morning, and the store cannot tell // those apart — so it reports the fact and lets the operator supply the intent. // // Everything else takes its level from what the fault costs. A credential the endpoint rejected // stops that integration dead; an address that does not answer might be a host still booting. function vv_ai_finding_severity(array $f): string { $key = (string)($f['conf_key'] ?? ''); // Deliberate-state findings never escalate, whatever their kind. if (vv_ai_conf_is_toggle($key)) return 'warn'; return match ($f['kind'] ?? '') { 'auth_rejected' => 'error', // answered and refused — nothing gets through until fixed 'missing_value' => 'error', // configured to use something that was never supplied 'unreachable' => 'warn', // may be transient; the strike system is what escalates it 'unknown_target' => 'warn', default => 'warn', }; } // ── Gates ──────────────────────────────────────────────────────────────────────────────────── // Two switches, because detecting and repairing are separate things to trust. // // AI_REPAIR_ENABLED alone gives a system that reads logs, files findings and offers fixes, and // writes nothing. That is the state this should live in first — long enough to read what it // found and disagree with some of it. A subsystem that starts by editing conf has to be believed // before there is any evidence for believing it. // // AI_REPAIR_AUTOFIX_ENABLED is what lets a proven value be written without being asked, and it // is meaningless on its own: nothing to write if nothing is looking. Both must be true, in the // same layered way AI_ENABLED is necessary but never sufficient. function vv_ai_repair_enabled(): bool { if (!vv_ai_config()['enabled']) return false; return strtolower(trim((string)(vv_conf_vars()['AI_REPAIR_ENABLED'] ?? 'false'))) === 'true'; } function vv_ai_repair_autofix_enabled(): bool { if (!vv_ai_repair_enabled()) return false; return strtolower(trim((string)(vv_conf_vars()['AI_REPAIR_AUTOFIX_ENABLED'] ?? 'false'))) === 'true'; } function vv_ai_findings_dir(): string { $d = AI_DATA_DIR . '/ai_findings'; if (!is_dir($d)) @mkdir($d, 0755, true); return $d; } function vv_ai_finding_retain_days(): int { $n = (int)(vv_conf_vars()['AI_FINDING_RETAIN_DAYS'] ?? 90); return max(1, $n); } // kind + subject + conf key. Deliberately not the message: the same wrong port produces slightly // different log text as the software around it changes, and that must not mint a second record. function vv_ai_finding_id(string $kind, string $subject, string $confKey): string { return substr(sha1(strtolower($kind . '|' . $subject . '|' . $confKey)), 0, 12); } function vv_ai_finding_path(string $id): ?string { if (!preg_match('/^[0-9a-f]{12}$/', $id)) return null; return vv_ai_findings_dir() . '/' . $id . '.json'; } function vv_ai_finding_get(string $id): ?array { $p = vv_ai_finding_path($id); if ($p === null || !is_file($p)) return null; $r = json_decode((string)@file_get_contents($p), true); return is_array($r) ? $r : null; } // Files a finding, or increments the one already describing this fault. // // $f expects: kind, subject, conf_key, conf_file, observed, evidence, source_log // and optionally: proposed, proven, state, note function vv_ai_finding_write(array $f): array { $kind = (string)($f['kind'] ?? ''); $subject = trim((string)($f['subject'] ?? '')); $confKey = trim((string)($f['conf_key'] ?? '')); $evidence = trim((string)($f['evidence'] ?? '')); if (!isset(VV_AI_FINDING_KINDS[$kind])) return ['ok' => false, 'error' => 'unknown kind']; if ($subject === '' || $confKey === '') return ['ok' => false, 'error' => 'subject and conf_key required']; if ($evidence === '') return ['ok' => false, 'error' => 'evidence required']; // The key has to be a real shell identifier for the same reason the conf writer insists on // it: a finding is a proposal to edit that key, and a malformed one can never be actioned. if (!vv_conf_key_valid($confKey)) return ['ok' => false, 'error' => 'malformed conf key']; $state = (string)($f['state'] ?? 'open'); if (!isset(VV_AI_FINDING_STATES[$state])) $state = 'open'; $now = time(); $id = vv_ai_finding_id($kind, $subject, $confKey); $rec = [ 'id' => $id, 'kind' => $kind, 'subject' => mb_substr($subject, 0, 120), 'conf_key' => $confKey, 'conf_file' => (string)($f['conf_file'] ?? 'master.conf'), // Secrets never enter this store. A finding about a rejected API key is about the key // being wrong, and the wrong value is of no use to anyone reading the record later. 'observed' => vv_conf_key_is_secret($confKey) ? '' : mb_substr((string)($f['observed'] ?? ''), 0, 300), 'proposed' => isset($f['proposed']) && !vv_conf_key_is_secret($confKey) ? mb_substr((string)$f['proposed'], 0, 300) : null, 'proven' => (bool)($f['proven'] ?? false), 'state' => $state, 'evidence' => mb_substr(vv_ai_redact($evidence), 0, 1200), 'source_log' => mb_substr((string)($f['source_log'] ?? ''), 0, 200), 'note' => mb_substr((string)($f['note'] ?? ''), 0, 1000), // Recomputed on every sighting rather than stored once: a key that becomes a toggle, or // a toggle that is replaced by a real value, changes what this finding means. 'severity' => vv_ai_finding_severity(['kind' => $kind, 'conf_key' => $confKey]), 'host' => vv_detect_host(), 'first' => $now, 'last' => $now, 'seen' => 1, 'closed_at' => null, // What the key read when the operator acknowledged it. Null unless acked; the ack // expires the moment the live value stops matching this. 'ack_value' => null, ]; $p = vv_ai_finding_path($id); if ($p === null) return ['ok' => false, 'error' => 'bad id']; if (is_file($p)) { $old = json_decode((string)@file_get_contents($p), true); if (is_array($old)) { $rec['first'] = $old['first'] ?? $now; $rec['seen'] = (int)($old['seen'] ?? 0) + 1; // A dismissed finding stays dismissed however many times the log repeats it — // otherwise "this is fine, stop telling me" lasts exactly one cycle. A fixed one // reopens, because seeing the fault again after a repair means the repair did not // hold, which is the single most important thing this store can tell anyone. if (($old['state'] ?? '') === 'dismissed') { $rec['state'] = 'dismissed'; $rec['closed_at'] = $old['closed_at'] ?? null; } // An acknowledgement holds only while the thing acknowledged is still true. Compare // the live value against what it read when the ack was given: unchanged means stay // quiet, changed means the note is stale and the finding comes back by itself. if (($old['state'] ?? '') === 'acknowledged') { $ackedAt = (string)($old['ack_value'] ?? ''); if ($ackedAt === (string)(vv_conf_vars()[$confKey] ?? '')) { $rec['state'] = 'acknowledged'; $rec['ack_value'] = $ackedAt; $rec['closed_at'] = $old['closed_at'] ?? null; } // Otherwise $rec keeps the state this sighting computed — it has reopened. } // Preserve an operator's note over a generated one. if ($rec['note'] === '' && !empty($old['note'])) $rec['note'] = $old['note']; } } if (@file_put_contents($p, json_encode($rec, JSON_PRETTY_PRINT)) === false) { return ['ok' => false, 'error' => 'write failed']; } return ['ok' => true, 'id' => $id, 'seen' => $rec['seen'], 'state' => $rec['state']]; } // $states filters; empty means everything. Newest activity first, because a finding seen in the // last cycle matters more than one that has been sitting fixed for a month. function vv_ai_findings_list(array $states = ['open', 'needs_operator']): array { $out = []; foreach ((array)@glob(vv_ai_findings_dir() . '/*.json') as $file) { $r = json_decode((string)@file_get_contents($file), true); if (!is_array($r)) continue; if ($states && !in_array($r['state'] ?? 'open', $states, true)) continue; $out[] = $r; } usort($out, fn($a, $b) => ($b['last'] ?? 0) <=> ($a['last'] ?? 0)); return $out; } function vv_ai_finding_set_state(string $id, string $state, string $note = ''): bool { if (!isset(VV_AI_FINDING_STATES[$state])) return false; $r = vv_ai_finding_get($id); if ($r === null) return false; $r['state'] = $state; $r['closed_at'] = in_array($state, ['open', 'needs_operator'], true) ? null : time(); if ($note !== '') $r['note'] = mb_substr(vv_ai_redact($note), 0, 1000); $p = vv_ai_finding_path($id); return $p !== null && @file_put_contents($p, json_encode($r, JSON_PRETTY_PRINT)) !== false; } function vv_ai_finding_close(string $id, string $note = ''): bool { return vv_ai_finding_set_state($id, 'resolved', $note); } function vv_ai_finding_dismiss(string $id, string $note = ''): bool { return vv_ai_finding_set_state($id, 'dismissed', $note); } // "I know about this — leave it, and tell me if it changes." // // Stamps the key's current value onto the record. Every later sighting compares against that // stamp, so the acknowledgement covers this state and not the key forever. Acking that critical // rsync is off says nothing about critical rsync being on. function vv_ai_finding_ack(string $id, string $note = ''): bool { $r = vv_ai_finding_get($id); if ($r === null) return false; $r['state'] = 'acknowledged'; $r['ack_value'] = (string)(vv_conf_vars()[$r['conf_key'] ?? ''] ?? ''); $r['closed_at'] = time(); if ($note !== '') $r['note'] = mb_substr(vv_ai_redact($note), 0, 1000); $p = vv_ai_finding_path($id); return $p !== null && @file_put_contents($p, json_encode($r, JSON_PRETTY_PRINT)) !== false; } // What the operator can do about a finding, and what each choice means. Returned rather than // hardcoded in the UI so the chat and the page cannot offer different options for the same row. // // Fix appears for anything with a proposed value, toggle or not — the prohibition is on the // sweep choosing, never on the operator choosing. Everything carries ack and cancel, because // "I know" and "not now" are always valid answers to being told something. function vv_ai_finding_actions(array $f): array { $actions = []; if (($f['proposed'] ?? null) !== null) { $actions['fix'] = vv_ai_conf_is_toggle((string)($f['conf_key'] ?? '')) ? 'Set ' . $f['conf_key'] . ' — a toggle, so this only ever happens because you asked' : 'Write the proven value to ' . $f['conf_key']; } $actions['ack'] = 'Known and intended. Stays quiet until ' . ($f['conf_key'] ?? 'it') . ' changes'; $actions['cancel'] = 'Leave it alone for now'; return $actions; } // Closed findings are kept for a while because "this happened before and here is what fixed it" // is worth more than the disk it costs. Open ones are never pruned — an unresolved problem does // not stop mattering because it is old. function vv_ai_findings_prune(): int { $cutoff = time() - (vv_ai_finding_retain_days() * 86400); $n = 0; foreach ((array)@glob(vv_ai_findings_dir() . '/*.json') as $file) { $r = json_decode((string)@file_get_contents($file), true); if (!is_array($r)) continue; if (in_array($r['state'] ?? 'open', ['open', 'needs_operator'], true)) continue; if ((int)($r['closed_at'] ?? 0) > $cutoff) continue; if (@unlink($file)) $n++; } return $n; } // What the assistant should raise when a page loads: things that need the operator, newest // first. Repaired findings are deliberately not here — a fix that worked is a log entry, not a // conversation, and opening every session with a list of things that already went right is how // an operator learns to close the panel without reading it. function vv_ai_findings_for_chat(int $limit = 3): array { return array_slice(vv_ai_findings_list(['needs_operator']), 0, max(1, $limit)); } // ── The sweep ──────────────────────────────────────────────────────────────────────────────── // There is no post-run hook in Varaverk — nothing fires when a job finishes. Rather than add a // call to forty scripts, this picks up run records that completed since the last pass. One entry // in an orchestrator's list instead of forty edits, and it batches naturally. // // Runs that reported ok are read too. A container failing its HTTP check warns and leaves the // watchdog exiting 0, so "only look at failures" would miss the whole class of fault this exists // for: the job worked, and told you something is wrong. function vv_ai_sweep_marker_path(): string { return STATE_DIR . '/ai_repair_sweep.db'; } function vv_ai_sweep_last(): int { return (int)trim((string)@file_get_contents(vv_ai_sweep_marker_path())); } function vv_ai_sweep_mark(int $ts): void { if (!is_dir(STATE_DIR)) @mkdir(STATE_DIR, 0755, true); @file_put_contents(vv_ai_sweep_marker_path(), (string)$ts, LOCK_EX); } // Run records that finished after $since. A record still marked running is skipped rather than // read half-written — it will be picked up on the pass after it finishes. function vv_ai_recent_runs(int $since): array { $out = []; $base = realpath(LOG_DIR); if ($base === false) return []; foreach ((array)@glob($base . '/{,*/,*/*/,*/*/*/}*.json', GLOB_BRACE) as $path) { $r = json_decode((string)@file_get_contents($path), true); if (!is_array($r) || empty($r['id']) || ($r['status'] ?? '') === 'running') continue; $end = (int)($r['end'] ?? 0); if ($end <= $since) continue; $log = preg_replace('/\.json$/', '.log', $path); if (!is_file($log)) continue; $out[] = ['id' => (string)$r['id'], 'status' => (string)($r['status'] ?? '?'), 'start' => (int)($r['start'] ?? 0), 'end' => $end, 'log' => $log]; } usort($out, fn($a, $b) => $a['end'] <=> $b['end']); return $out; } // The lines one run wrote, and only those. Logs are appended across runs, so a tail alone would // re-read the previous run's output and re-report faults that have already been dealt with. // Filtering on the leading timestamp scopes the evidence to the run being examined. function vv_ai_run_log_lines(string $logPath, int $startTs, int $maxLines = 2000): array { $out = []; $rc = 0; @exec('tail -n ' . (int)$maxLines . ' ' . escapeshellarg($logPath) . ' 2>/dev/null', $out, $rc); if ($rc !== 0) return []; $kept = []; foreach ($out as $line) { if (preg_match('/^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})/', $line, $m)) { // A line older than the run belongs to a previous one. Two seconds of slack because // the record's start is stamped by the runner, not by the first line the job writes. if (strtotime($m[1]) < $startTs - 2) continue; } $kept[] = $line; } return $kept; } // One pass. Returns a summary rather than logging it, so the caller decides what to record and // the whole thing stays testable without a log to read afterwards. // // $dryRun does everything except write conf and move the marker — including probing, which is // the point: it answers "what would this have done" with real evidence rather than a guess. function vv_ai_repair_sweep(bool $dryRun = false): array { if (!vv_ai_repair_enabled()) { return ['ok' => false, 'error' => 'AI_REPAIR_ENABLED is not true', 'runs' => 0]; } $started = time(); $since = vv_ai_sweep_last(); $runs = vv_ai_recent_runs($since); $sum = ['ok' => true, 'runs' => count($runs), 'findings' => 0, 'fixed' => 0, 'needs_operator' => 0, 'resolved' => 0, 'quiet' => 0, 'details' => []]; foreach ($runs as $run) { $lines = vv_ai_run_log_lines($run['log'], $run['start']); if (!$lines) continue; $rel = ltrim(str_replace(realpath(LOG_DIR), '', $run['log']), '/'); foreach (vv_ai_triage_log($lines, $rel) as $cand) { $cand = vv_ai_probe_finding($cand); $sum['findings']++; // Write first, so a finding exists even if the repair below fails. A repair that // errored without leaving a record is the one failure mode there is no way back from. $w = vv_ai_finding_write($cand); if (!($w['ok'] ?? false)) continue; $id = $w['id']; // Already acknowledged or dismissed — the operator has spoken, and re-fixing behind // them would be the opposite of what an acknowledgement means. if (in_array($w['state'] ?? '', ['acknowledged', 'dismissed'], true)) { $sum['quiet']++; continue; } if (($cand['state'] ?? '') === 'resolved') { vv_ai_finding_close($id, (string)($cand['note'] ?? '')); $sum['resolved']++; continue; } if (vv_ai_finding_may_autofix($cand)) { if ($dryRun) { $sum['details'][] = "would fix {$cand['conf_key']} → {$cand['proposed']}"; continue; } $r = vv_ai_finding_apply_action($id, 'fix', 'Probed and written by the repair sweep.'); if ($r['ok'] ?? false) { $sum['fixed']++; $sum['details'][] = "fixed {$cand['conf_key']}"; } else { $sum['needs_operator']++; vv_ai_finding_set_state($id, 'needs_operator', (string)($r['error'] ?? '')); } continue; } if (($cand['state'] ?? '') === 'needs_operator') $sum['needs_operator']++; } } // Marked only on a completed pass, and to when the pass began — a job that finished while // this was running is then picked up next time instead of being skipped for having ended // before a marker written at the end. if (!$dryRun) vv_ai_sweep_mark($started); return $sum; } // ── Answering a finding in words ───────────────────────────────────────────────────────────── // The buttons are unambiguous by construction. This is for the other path — replying "yeah go // ahead" in the chat that raised the finding — and it is matched here rather than asked of the // model, because the model's answer would be a conf write and a wrong reading of "no, leave it" // is not recoverable by apologising. // // Same shape as vv_ai_route_from_chat(): anchored patterns, most specific first, and anything // unrecognised returns null so the assistant asks again instead of guessing. Two actions both // matching is also null — "leave it, I know" and "leave it for now" differ by one clause and // mean different things, so a phrase that supports both is not an instruction yet. const VV_AI_ACTION_PATTERNS = [ // Acknowledge — "this is deliberate, stop telling me". 'ack' => [ '/\b(i|we) know\b/u', '/\b(that|this|it)(?:\'s| is) (fine|expected|intentional|deliberate|on purpose)\b/u', '/\bon purpose\b/u', '/\b(aware|acknowledge|ack)\b/u', '/\bmeant to be\b/u', ], // Apply the proposed value. 'fix' => [ '/\bfix (it|that|this|them)?\b/u', '/\b(go ahead|do it|apply|make the change|change it|update it|correct it)\b/u', // A bare affirmative, as the whole message — "yes" answering "shall I fix it" is an // instruction, "yes it looks wrong" is agreement about the diagnosis and nothing more. // A trailing please is still bare. '/\b(yes|yeah|yep|yup|sure|ok|okay)\b(\s*,?\s*please)?[\s,.!]*$/u', '/\bplease do\b/u', ], // Not now — no state written, it comes back next sweep. 'cancel' => [ '/\b(not now|later|leave it (alone|for now)|skip( it)?|cancel|ignore for now)\b/u', '/\b(no|nope|nah)\b[\s,.!]*$/u', '/\b(don\'?t|do not) (fix|touch|change|write|apply)\b/u', ], ]; // Returns 'fix' | 'ack' | 'cancel', or null when the reply does not clearly mean one of them. // // Only call this when a finding is actually pending. A bare "yes" means fix in answer to "shall // I fix it" and means nothing at all on its own, and the difference is context this function // cannot see. function vv_ai_finding_action_from_text(string $text): ?string { $t = strtolower(trim($text)); if ($t === '') return null; $t = preg_replace('/\s+/', ' ', $t); $matched = []; foreach (VV_AI_ACTION_PATTERNS as $action => $patterns) { foreach ($patterns as $re) { if (preg_match($re, $t)) { $matched[$action] = true; break; } } } // Exactly one reading, or none. "leave it, I know" hits both ack and cancel; that is a // sentence the operator should be asked to restate, not one to pick a winner from. return count($matched) === 1 ? array_key_first($matched) : null; } // Carry out an answered action against a stored finding. // // Fix goes through the same guarded write path as everything else, and is the one place a // toggle may be written — because reaching here means the operator asked for it by name. The // unattended sweep never calls this. function vv_ai_finding_apply_action(string $id, string $action, string $note = ''): array { $f = vv_ai_finding_get($id); if ($f === null) return ['ok' => false, 'error' => 'no such finding']; switch ($action) { case 'ack': return ['ok' => vv_ai_finding_ack($id, $note), 'action' => 'ack']; case 'cancel': // Deliberately writes nothing at all. "Not now" is not a state, it is the absence of // one — recording it would make the finding look decided when it is still open. return ['ok' => true, 'action' => 'cancel']; case 'fix': $proposed = $f['proposed'] ?? null; if ($proposed === null) return ['ok' => false, 'error' => 'nothing proposed to write']; $key = (string)$f['conf_key']; $ok = vv_conf_write_changes([[ 'file' => (string)($f['conf_file'] ?? 'master.conf'), 'key' => $key, 'value' => (string)$proposed, 'type' => 'scalar', ]]); $wrote = !in_array(false, $ok, true); if ($wrote) { vv_ai_finding_set_state($id, 'fixed', $note !== '' ? $note : 'Wrote ' . $key . ' at the operator\'s request.'); } return ['ok' => $wrote, 'action' => 'fix', 'error' => $wrote ? null : 'conf write refused — see conf_changes.log']; } return ['ok' => false, 'error' => 'unknown action']; } // ── Resolving a log line back to a conf key ────────────────────────────────────────────────── // By value wherever possible, by name only as a fallback. // // A log line usually contains the thing that failed — the URL that did not answer. That value // came from a conf key, so searching the conf for which key holds it is an exact lookup with a // definite answer. Guessing the key from the container's name is inference, and the failure mode // is silent: HOST1_EMBY_URL and HOST1_EMBY_EXTERNAL_URL are both plausible for "Emby" and only // one of them is the value that just failed. // // Same principle as resolve_tailscale_ip() refusing similarity matching for host identity: an // exact match or an honest nothing. // Every conf key whose value the given text starts with, longest first. Prefix rather than // equality because a log reports the URL it actually called — the conf value plus an endpoint // path — and the longest match is the most specific key that could have produced it. function vv_ai_conf_keys_for_value(string $value): array { $value = trim($value); if ($value === '' || strlen($value) < 6) return []; $hits = []; foreach (vv_conf_vars() as $k => $v) { $v = trim((string)$v); if ($v === '' || strlen($v) < 6) continue; if ($v === $value || str_starts_with($value, $v)) $hits[$k] = strlen($v); } arsort($hits); return array_keys($hits); } // __, built literally and then checked for existence. Nothing is // inferred: either the conf holds a key by exactly that name or this returns null. function vv_ai_conf_key_for_subject(string $subject, string $suffix): ?string { $norm = strtoupper(preg_replace('/[^A-Za-z0-9]+/', '_', trim($subject))); if ($norm === '') return null; $key = strtoupper(vv_detect_host()) . '_' . $norm . '_' . strtoupper($suffix); return array_key_exists($key, vv_conf_vars()) ? $key : null; } // Which conf file a key lives in. A finding has to name the file it would be edited in, and // host keys are not in master.conf. function vv_ai_conf_file_for_key(string $key): string { foreach (vv_get_conf_files() as $f) { if (preg_match('/^\s*' . preg_quote($key, '/') . '\s*=/m', vv_read_conf_raw($f))) return $f; } return 'master.conf'; } // ── Deterministic triage ───────────────────────────────────────────────────────────────────── // Patterns are written against log formats that exist in this repo today, taken from the emit // sites rather than imagined. A pattern that stops matching because its log line was reworded // produces no finding, which is the safe direction — the alternative is a pattern loose enough // to match anything, which fills the store with rows nobody can act on. // // No model runs here. "Connection refused" is not a judgement call, and a 14B model invoked // after every cron job to notice it would be slower, costlier and less reliable than a regex. // The model's job starts where these stop: explaining a finding, and talking the operator // through the ones that cannot be repaired automatically. const VV_AI_TRIAGE_PATTERNS = [ // docker_watchdog HTTP check — the richest signal, carrying both subject and failing URL. [ 're' => '/^(?P\S+) — not responding at (?P\S+) \(strike/u', 'kind' => 'unreachable', 'suffix' => 'URL', ], // docker_watchdog API check, 401/403. The endpoint answered, so the address is right and // the credential is not. [ 're' => '/^(?P\S+) — API check skipped \(HTTP (?:401|403)/u', 'kind' => 'auth_rejected', 'suffix' => 'API_KEY', ], // Configured for an API check with nothing to authenticate with. [ 're' => '/^(?P\S+) — API check skipped \(no key configured\)/u', 'kind' => 'missing_value', 'suffix' => 'API_KEY', ], [ 're' => '/^Skipping (?P\S+) — placeholder API key/u', 'kind' => 'missing_value', 'suffix' => 'API_KEY', ], [ 're' => '/^(?P\S+) (?:—\s*)?API unreachable/u', 'kind' => 'unreachable', 'suffix' => 'URL', ], ]; // One log line in, at most one finding candidate out. Returns null for everything else, which is // almost every line. function vv_ai_triage_line(string $line): ?array { // Strip the timestamp and level decoration the logger adds, so patterns can anchor on ^. $body = preg_replace('/^\S+\s+\S+\s+(?:[^\[]*\[[A-Z]+\]\s*)?/u', '', rtrim($line)); $body = trim((string)$body); if ($body === '') return null; foreach (VV_AI_TRIAGE_PATTERNS as $p) { if (!preg_match($p['re'], $body, $m)) continue; $subject = trim($m['subject'] ?? ''); $observed = trim($m['observed'] ?? ''); if ($subject === '') continue; // Value first, name second. Only one candidate key is accepted — two keys holding the // same value means the log cannot say which one produced it, and picking either is the // guess this whole approach exists to avoid. $key = null; if ($observed !== '') { $byValue = vv_ai_conf_keys_for_value($observed); if (count($byValue) === 1) $key = $byValue[0]; } if ($key === null) $key = vv_ai_conf_key_for_subject($subject, $p['suffix']); if ($key === null) continue; // nothing actionable — no row return [ 'kind' => $p['kind'], 'subject' => $subject, 'conf_key' => $key, 'conf_file' => vv_ai_conf_file_for_key($key), 'observed' => $observed !== '' ? $observed : (string)(vv_conf_vars()[$key] ?? ''), 'evidence' => $body, ]; } return null; } // A whole log tail in, one candidate per distinct fault out. A job that retried twelve times // produces twelve identical lines, and the store's dedupe would collapse them anyway — doing it // here keeps the sweep from writing the same file twelve times in a row. function vv_ai_triage_log(array $lines, string $sourceLog = ''): array { $found = []; foreach ($lines as $line) { $c = vv_ai_triage_line((string)$line); if ($c === null) continue; $c['source_log'] = $sourceLog; $found[vv_ai_finding_id($c['kind'], $c['subject'], $c['conf_key'])] = $c; } return array_values($found); } // ── Proving a candidate ────────────────────────────────────────────────────────────────────── // The guard the whole unattended path rests on: nothing is written that has not answered. // // A model can be confident that a port should be 8686. A probe can report that 8686 answered. // Only the second is a fact, and only facts get written to conf without being asked. Everything // a probe cannot settle becomes a conversation instead — which is not a lesser outcome, it is // the honest one for a value that cannot be derived from this machine. // // Deliberately narrow. These check reachability and identity, never correctness of behaviour: // that Lidarr answers on 8686 does not prove 8686 is the port you meant, only that something is // listening there and calling itself Lidarr. Proving intent is not a probe's job. function vv_ai_probe_timeout(): int { return max(1, (int)(vv_conf_vars()['AI_PROBE_TIMEOUT'] ?? 4)); } // Does this URL answer at all? Any HTTP status counts, including 401 — a refusal is proof that // something is listening and speaking HTTP, which is exactly what an address probe is asking. // Distinguishing "wrong address" from "wrong credential" is the point of having both kinds. function vv_ai_probe_url(string $url): array { if (!preg_match('#^https?://[^\s/$.?\#][^\s]*$#i', $url)) { return ['ok' => false, 'reason' => 'not a url']; } $ch = curl_init($url); curl_setopt_array($ch, [ CURLOPT_RETURNTRANSFER => true, CURLOPT_NOBODY => true, CURLOPT_TIMEOUT => vv_ai_probe_timeout(), CURLOPT_CONNECTTIMEOUT => vv_ai_probe_timeout(), CURLOPT_SSL_VERIFYPEER => false, // these are LAN and Tailscale endpoints, often self-signed CURLOPT_SSL_VERIFYHOST => false, ]); curl_exec($ch); $code = (int)curl_getinfo($ch, CURLINFO_RESPONSE_CODE); $err = curl_error($ch); curl_close($ch); return $code > 0 ? ['ok' => true, 'code' => $code] : ['ok' => false, 'reason' => $err !== '' ? $err : 'no response']; } // Every container on this host, by exact name. The membership test for unknown_target findings, // and one docker call rather than one per candidate. function vv_ai_container_names(): array { static $names = null; if ($names !== null) return $names; $out = []; @exec('timeout ' . vv_ai_probe_timeout() . " docker ps -a --format '{{.Names}}' 2>/dev/null", $out, $rc); return $names = ($rc === 0) ? array_values(array_filter(array_map('trim', $out))) : []; } // Candidate corrections for a URL whose host or port stopped answering. // // Only two transformations, both conservative: the same host on a port that some other conf key // already uses, and the same port on a host some other conf key already names. Both draw // exclusively from values already present in this installation's conf — nothing is invented, and // a scan of the port range is deliberately not attempted. Finding *a* listening port is not the // same as finding the right service, and a probe that accepts any answer would happily point // Lidarr at Sonarr. function vv_ai_url_candidates(string $observed): array { $parts = @parse_url($observed); if (!is_array($parts) || empty($parts['host'])) return []; $hosts = $ports = []; foreach (vv_conf_vars() as $k => $v) { $v = trim((string)$v); if (!preg_match('#^https?://#i', $v)) continue; $p = @parse_url($v); if (!is_array($p) || empty($p['host'])) continue; $hosts[$p['host']] = true; if (!empty($p['port'])) $ports[(int)$p['port']] = true; } $scheme = $parts['scheme'] ?? 'http'; $path = $parts['path'] ?? ''; $out = []; foreach (array_keys($ports) as $port) { $c = $scheme . '://' . $parts['host'] . ':' . $port . $path; if ($c !== $observed) $out[$c] = true; } foreach (array_keys($hosts) as $host) { $port = !empty($parts['port']) ? ':' . $parts['port'] : ''; $c = $scheme . '://' . $host . $port . $path; if ($c !== $observed) $out[$c] = true; } return array_keys($out); } // Try to prove a correction for one finding. Returns the finding with 'proposed' and 'proven' // filled in, or unchanged when nothing could be proven — which is the common case and not a // failure. // // auth_rejected and missing_value are never proven here on purpose. A credential cannot be // derived from this host by definition: if it could be read from somewhere, it would not be a // credential. Those go straight to the operator. function vv_ai_probe_finding(array $f): array { $kind = (string)($f['kind'] ?? ''); if ($kind === 'unknown_target') { $observed = (string)($f['observed'] ?? ''); // Exact membership only. A container name is a literal, and "close to an existing name" // is how a repair renames the wrong thing. if (in_array($observed, vv_ai_container_names(), true)) { return $f; // it exists after all — nothing to correct } return $f + ['state' => 'needs_operator']; } if ($kind !== 'unreachable') { // Nothing on this machine can supply a credential, so there is nothing to prove. $f['state'] = 'needs_operator'; return $f; } $observed = (string)($f['observed'] ?? ''); // If the observed address answers now, the fault has cleared on its own — a host that was // rebooting, most often. Recording a proposal here would repair something already working. if (vv_ai_probe_url($observed)['ok']) { $f['state'] = 'resolved'; $f['note'] = 'Answered when probed — the address was reachable again by the time this ran.'; return $f; } $answered = []; foreach (vv_ai_url_candidates($observed) as $candidate) { if (vv_ai_probe_url($candidate)['ok']) $answered[] = $candidate; } // Exactly one, or none. Two addresses answering means the probe cannot say which is the // right one, and picking either is the guess this exists to prevent. if (count($answered) === 1) { $f['proposed'] = $answered[0]; $f['proven'] = true; $f['note'] = 'Probed ' . $answered[0] . ' and it answered; ' . $observed . ' did not.'; return $f; } $f['proven'] = false; $f['state'] = 'needs_operator'; $f['note'] = $answered ? 'Several addresses answered (' . implode(', ', $answered) . '), so none was written.' : 'Nothing answered at ' . $observed . ', and no address in the conf answered either.'; return $f; }