Sweep finished runs for misconfigurations instead of waiting to be told

Nothing in Varaverk fires when a job ends, so this picks up run records that completed since the last pass rather than adding a hook to forty scripts. Ahead of stability in the cycle on purpose: a wrong port is not fixed by rebooting the machine.
2026-08-09 19:38:02 -04:00
parent 955f50b94e
commit e91a74d31f
5 changed files with 319 additions and 0 deletions
@@ -383,6 +383,137 @@ function vv_ai_findings_for_chat(int $limit = 3): array {
    return array_slice(vv_ai_findings_list(['needs_operator']), 0, max(1, $limit));
 }

+// ── The sweep ────────────────────────────────────────────────────────────────────────────────
+// There is no post-run hook in Varaverk — nothing fires when a job finishes. Rather than add a
+// call to forty scripts, this picks up run records that completed since the last pass. One entry
+// in an orchestrator's list instead of forty edits, and it batches naturally.
+//
+// Runs that reported ok are read too. A container failing its HTTP check warns and leaves the
+// watchdog exiting 0, so "only look at failures" would miss the whole class of fault this exists
+// for: the job worked, and told you something is wrong.
+
+function vv_ai_sweep_marker_path(): string {
+    return STATE_DIR . '/ai_repair_sweep.db';
+}
+
+function vv_ai_sweep_last(): int {
+    return (int)trim((string)@file_get_contents(vv_ai_sweep_marker_path()));
+}
+
+function vv_ai_sweep_mark(int $ts): void {
+    if (!is_dir(STATE_DIR)) @mkdir(STATE_DIR, 0755, true);
+    @file_put_contents(vv_ai_sweep_marker_path(), (string)$ts, LOCK_EX);
+}
+
+// Run records that finished after $since. A record still marked running is skipped rather than
+// read half-written — it will be picked up on the pass after it finishes.
+function vv_ai_recent_runs(int $since): array {
+    $out  = [];
+    $base = realpath(LOG_DIR);
+    if ($base === false) return [];
+
+    foreach ((array)@glob($base . '/{,*/,*/*/,*/*/*/}*.json', GLOB_BRACE) as $path) {
+        $r = json_decode((string)@file_get_contents($path), true);
+        if (!is_array($r) || empty($r['id']) || ($r['status'] ?? '') === 'running') continue;
+
+        $end = (int)($r['end'] ?? 0);
+        if ($end <= $since) continue;
+
+        $log = preg_replace('/\.json$/', '.log', $path);
+        if (!is_file($log)) continue;
+
+        $out[] = ['id' => (string)$r['id'], 'status' => (string)($r['status'] ?? '?'),
+                  'start' => (int)($r['start'] ?? 0), 'end' => $end, 'log' => $log];
+    }
+    usort($out, fn($a, $b) => $a['end'] <=> $b['end']);
+    return $out;
+}
+
+// The lines one run wrote, and only those. Logs are appended across runs, so a tail alone would
+// re-read the previous run's output and re-report faults that have already been dealt with.
+// Filtering on the leading timestamp scopes the evidence to the run being examined.
+function vv_ai_run_log_lines(string $logPath, int $startTs, int $maxLines = 2000): array {
+    $out = []; $rc = 0;
+    @exec('tail -n ' . (int)$maxLines . ' ' . escapeshellarg($logPath) . ' 2>/dev/null', $out, $rc);
+    if ($rc !== 0) return [];
+
+    $kept = [];
+    foreach ($out as $line) {
+        if (preg_match('/^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})/', $line, $m)) {
+            // A line older than the run belongs to a previous one. Two seconds of slack because
+            // the record's start is stamped by the runner, not by the first line the job writes.
+            if (strtotime($m[1]) < $startTs - 2) continue;
+        }
+        $kept[] = $line;
+    }
+    return $kept;
+}
+
+// One pass. Returns a summary rather than logging it, so the caller decides what to record and
+// the whole thing stays testable without a log to read afterwards.
+//
+// $dryRun does everything except write conf and move the marker — including probing, which is
+// the point: it answers "what would this have done" with real evidence rather than a guess.
+function vv_ai_repair_sweep(bool $dryRun = false): array {
+    if (!vv_ai_repair_enabled()) {
+        return ['ok' => false, 'error' => 'AI_REPAIR_ENABLED is not true', 'runs' => 0];
+    }
+
+    $started = time();
+    $since   = vv_ai_sweep_last();
+    $runs    = vv_ai_recent_runs($since);
+
+    $sum = ['ok' => true, 'runs' => count($runs), 'findings' => 0, 'fixed' => 0,
+            'needs_operator' => 0, 'resolved' => 0, 'quiet' => 0, 'details' => []];
+
+    foreach ($runs as $run) {
+        $lines = vv_ai_run_log_lines($run['log'], $run['start']);
+        if (!$lines) continue;
+
+        $rel = ltrim(str_replace(realpath(LOG_DIR), '', $run['log']), '/');
+        foreach (vv_ai_triage_log($lines, $rel) as $cand) {
+            $cand = vv_ai_probe_finding($cand);
+            $sum['findings']++;
+
+            // Write first, so a finding exists even if the repair below fails. A repair that
+            // errored without leaving a record is the one failure mode there is no way back from.
+            $w = vv_ai_finding_write($cand);
+            if (!($w['ok'] ?? false)) continue;
+            $id = $w['id'];
+
+            // Already acknowledged or dismissed — the operator has spoken, and re-fixing behind
+            // them would be the opposite of what an acknowledgement means.
+            if (in_array($w['state'] ?? '', ['acknowledged', 'dismissed'], true)) {
+                $sum['quiet']++;
+                continue;
+            }
+
+            if (($cand['state'] ?? '') === 'resolved') {
+                vv_ai_finding_close($id, (string)($cand['note'] ?? ''));
+                $sum['resolved']++;
+                continue;
+            }
+
+            if (vv_ai_finding_may_autofix($cand)) {
+                if ($dryRun) { $sum['details'][] = "would fix {$cand['conf_key']} → {$cand['proposed']}"; continue; }
+                $r = vv_ai_finding_apply_action($id, 'fix', 'Probed and written by the repair sweep.');
+                if ($r['ok'] ?? false) { $sum['fixed']++; $sum['details'][] = "fixed {$cand['conf_key']}"; }
+                else                   { $sum['needs_operator']++; vv_ai_finding_set_state($id, 'needs_operator', (string)($r['error'] ?? '')); }
+                continue;
+            }
+
+            if (($cand['state'] ?? '') === 'needs_operator') $sum['needs_operator']++;
+        }
+    }
+
+    // Marked only on a completed pass, and to when the pass began — a job that finished while
+    // this was running is then picked up next time instead of being skipped for having ended
+    // before a marker written at the end.
+    if (!$dryRun) vv_ai_sweep_mark($started);
+
+    return $sum;
+}
+
 // ── Answering a finding in words ─────────────────────────────────────────────────────────────
 // The buttons are unambiguous by construction. This is for the other path — replying "yeah go
 // ahead" in the chat that raised the finding — and it is matched here rather than asked of the