Sweep finished runs for misconfigurations instead of waiting to be told

Nothing in Varaverk fires when a job ends, so this picks up run records that completed since the last pass rather than adding a hook to forty scripts. Ahead of stability in the cycle on purpose: a wrong port is not fixed by rebooting the machine.
2026-08-09 19:38:02 -04:00
parent 955f50b94e
commit e91a74d31f
5 changed files with 319 additions and 0 deletions
@@ -403,8 +403,13 @@
        "Watchdogs/resource_watchdog.sh"   # reduce system pressure before healing attempts
        "Watchdogs/docker_watchdog.sh"     # heal containers with freed resources
        "Watchdogs/system_watchdog.sh"     # system component health — storage + webgui
+        "Plugin/unraid/Tools/ai_repair_sweep.sh"  # read what the last cycle logged; off unless AI_REPAIR_ENABLED
        "Watchdogs/stability_watchdog.sh"  # reboot if all else fails — last line of defense
    )
+# The repair sweep sits ahead of stability deliberately, which is the one exception to "stability
+# last". It reads the previous cycle's logs and may correct the very misconfiguration stability
+# would otherwise reboot for — a wrong port is not fixed by restarting the machine. It is bounded
+# by AI_PROBE_TIMEOUT, exits 0 in every case, and does nothing at all unless AI_REPAIR_ENABLED.

 # ━━━ System Watchdog ━━━
 # system_watchdog.sh runs SYSTEM_WATCHDOG_SCRIPTS sequentially each cycle.
@@ -0,0 +1,103 @@
+<?php
+// ═══════════════════════════════════════════════════════════════════════════════════════════════
+// PURPOSE
+//   One pass of the repair sweep. Reads the logs of jobs that finished since the last pass,
+//   turns known error shapes into findings, probes for a correction, and either writes a proven
+//   value or leaves the finding for the operator.
+//
+// OPERATIONAL MODEL
+//   Called from ai_repair_sweep.sh, which the watchdog orchestrator runs. Everything it does
+//   lives in include/ai_repair.php so the same code path is what the tests exercise; this file
+//   owns only the lock, the log line, and the exit code.
+//
+// DESIGN PRINCIPLES
+//   Non-fatal, always.
+//       Exits 0 on a disabled feature, a held lock, or a sweep that found nothing. Repair is an
+//       enhancement — the orchestrator that runs it has real work to do either side, and this
+//       must never be the reason a watchdog cycle reports failure.
+//
+//   The summary is logged, not the reasoning.
+//       What was scanned, what was found, what was written. The findings themselves are the
+//       record; duplicating their contents into a log would be two copies to keep in step.
+//
+// OPERATIONAL SAFEGUARDS
+//   One sweep at a time.
+//       flock, non-blocking. A pass that overruns its fifteen-minute slot must not have a second
+//       copy start probing and writing conf underneath it.
+//
+//   Nothing is written unless two switches say so.
+//       AI_REPAIR_ENABLED gates the sweep; AI_REPAIR_AUTOFIX_ENABLED gates writing. With only
+//       the first, this reads and files and changes no configuration at all.
+//
+// RUNTIME MODES
+//   ai_repair_sweep.php              one pass
+//   ai_repair_sweep.php --dry-run    probe and report, write nothing, leave the marker alone
+//   ai_repair_sweep.php --status     what the last pass did, and what is open
+// ═══════════════════════════════════════════════════════════════════════════════════════════════
+require_once dirname(__DIR__) . '/include/ai_repair.php';
+
+$args   = array_slice($argv ?? [], 1);
+$dryRun = in_array('--dry-run', $args, true);
+$status = in_array('--status', $args, true);
+
+function rlog(string $msg): void {
+    if (!is_dir(LOG_DIR)) return;
+    @file_put_contents(LOG_DIR . '/ai_repair.log',
+        date('Y-m-d H:i:s') . ' ' . $msg . "\n", FILE_APPEND | LOCK_EX);
+}
+
+if ($status) {
+    $last = vv_ai_sweep_last();
+    $open = vv_ai_findings_list(['open', 'needs_operator']);
+    printf("repair:   %s\n", vv_ai_repair_enabled() ? 'enabled' : 'disabled');
+    printf("autofix:  %s\n", vv_ai_repair_autofix_enabled() ? 'enabled' : 'disabled (detect only)');
+    printf("last pass: %s\n", $last ? date('Y-m-d H:i:s', $last) : 'never');
+    printf("open findings: %d\n", count($open));
+    foreach ($open as $f) {
+        printf("  [%s] %-22s %-28s seen %d — %s\n",
+            $f['severity'] ?? '?', $f['subject'] ?? '?', $f['conf_key'] ?? '?',
+            (int)($f['seen'] ?? 0), $f['state'] ?? '?');
+    }
+    exit(0);
+}
+
+if (!vv_ai_repair_enabled()) exit(0);   // silent: a disabled feature is not an event
+
+$lockPath = VV_CACHE_ROOT . '/ai_repair.lock';
+if (!is_dir(dirname($lockPath))) @mkdir(dirname($lockPath), 0755, true);
+$lock = @fopen($lockPath, 'c');
+if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) {
+    if ($lock) fclose($lock);
+    rlog('skipped — a sweep is already running');
+    exit(0);
+}
+
+try {
+    $t0  = microtime(true);
+    $sum = vv_ai_repair_sweep($dryRun);
+    $ms  = (int)round((microtime(true) - $t0) * 1000);
+
+    if (!($sum['ok'] ?? false)) {
+        rlog('sweep refused — ' . ($sum['error'] ?? 'unknown'));
+        exit(0);
+    }
+
+    // Nothing found and nothing to say. A line every fifteen minutes reporting no news is how a
+    // log stops being read.
+    if ($sum['findings'] === 0) {
+        if ($sum['runs'] > 0 && $dryRun) rlog(sprintf('dry-run: %d run(s), nothing found (%dms)', $sum['runs'], $ms));
+        exit(0);
+    }
+
+    rlog(sprintf('%s%d run(s): %d finding(s), %d fixed, %d for the operator, %d resolved, %d quiet (%dms)',
+        $dryRun ? 'dry-run: ' : '', $sum['runs'], $sum['findings'], $sum['fixed'],
+        $sum['needs_operator'], $sum['resolved'], $sum['quiet'], $ms));
+
+    foreach ($sum['details'] as $d) rlog('  ' . $d);
+
+} finally {
+    flock($lock, LOCK_UN);
+    fclose($lock);
+}
+
+exit(0);
@@ -0,0 +1,78 @@
+#!/bin/bash
+# ==============================================================================================
+# ================================== AI Repair Sweep ===========================================
+# ==============================================================================================
+# PURPOSE
+# ─────────────────────────────────────────────────────────────────────────────
+# Reads the logs of jobs that finished since the last pass, turns known error shapes into
+# findings, probes for a correction, and either writes a proven value or leaves the finding
+# for the operator to answer.
+# Runs from the watchdog orchestrator. Off unless AI_REPAIR_ENABLED is true.
+# ==============================================================================================
+# OPERATIONAL MODEL
+# ==============================================================================================
+# A one-line shim: exec php on ai_repair_sweep.php in the same directory.
+# The logic is PHP because everything it needs already is — the conf writer with its backups
+# and read-back verification, the findings store, and the probe layer are all functions the
+# WebGUI shares. A bash reimplementation would be a second conf writer, which is precisely the
+# drift the guarded write path exists to prevent.
+#
+# There is no post-run hook in Varaverk; nothing fires when a job finishes. The sweep picks up
+# completed run records instead, so this is one entry in an orchestrator list rather than a
+# call added to forty scripts.
+# ==============================================================================================
+# DESIGN PRINCIPLES
+# ==============================================================================================
+# A Shim, Not a Program
+#   This file exists only because the scheduler runs shell scripts and the work is PHP.
+#   Anything added here would be logic the WebGUI cannot reach, and the operator answering a
+#   finding in the browser must take exactly the same path as the sweep that filed it.
+#
+# Detecting And Repairing Are Separate Trusts
+#   AI_REPAIR_ENABLED alone reads logs, files findings and proposes fixes, writing nothing.
+#   AI_REPAIR_AUTOFIX_ENABLED is what allows a value to be written, and only ever one a probe
+#   has answered on. Both live in master.conf; neither is set by this script.
+# ==============================================================================================
+# OPERATIONAL SAFEGUARDS
+# ==============================================================================================
+# Never Fatal
+#   Always exits 0 — on a disabled feature, a held lock, or a failed pass. The watchdog
+#   orchestrator runs real work either side of this, and a repair sweep must never be the
+#   reason a cycle reports failure.
+#
+# One Sweep At A Time
+#   The PHP takes a non-blocking flock. A pass that overruns its slot cannot have a second
+#   copy start probing and writing conf underneath it.
+#
+# Nothing Is Written That Has Not Answered
+#   A value reaches conf only after a probe got a response from it. Toggles are never written
+#   unattended at all — whether something should be switched on is a decision about intent,
+#   and a probe cannot prove intent.
+# ==============================================================================================
+# CONFIGURATION
+# ==============================================================================================
+#
+# AI_ENABLED                 master switch; nothing here runs without it
+# AI_REPAIR_ENABLED          read logs and file findings
+# AI_REPAIR_AUTOFIX_ENABLED  allow a proven value to be written unattended
+# AI_PROBE_TIMEOUT           seconds a single probe may take
+# AI_FINDING_RETAIN_DAYS     how long closed findings are kept
+#
+# ==============================================================================================
+# RUNTIME MODES
+# ==============================================================================================
+#
+# ai_repair_sweep.sh
+#     One pass. Files findings, applies proven fixes if autofix is on.
+#
+# ai_repair_sweep.sh --dry-run
+#     Probes and reports what it would do. Writes no conf and does not move the marker, so the
+#     same runs are examined again next pass.
+#
+# ai_repair_sweep.sh --status
+#     Both switches, when the last pass ran, and every open finding.
+#
+# ==============================================================================================
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+php "$SCRIPT_DIR/ai_repair_sweep.php" "$@"
@@ -383,6 +383,137 @@ function vv_ai_findings_for_chat(int $limit = 3): array {
    return array_slice(vv_ai_findings_list(['needs_operator']), 0, max(1, $limit));
 }

+// ── The sweep ────────────────────────────────────────────────────────────────────────────────
+// There is no post-run hook in Varaverk — nothing fires when a job finishes. Rather than add a
+// call to forty scripts, this picks up run records that completed since the last pass. One entry
+// in an orchestrator's list instead of forty edits, and it batches naturally.
+//
+// Runs that reported ok are read too. A container failing its HTTP check warns and leaves the
+// watchdog exiting 0, so "only look at failures" would miss the whole class of fault this exists
+// for: the job worked, and told you something is wrong.
+
+function vv_ai_sweep_marker_path(): string {
+    return STATE_DIR . '/ai_repair_sweep.db';
+}
+
+function vv_ai_sweep_last(): int {
+    return (int)trim((string)@file_get_contents(vv_ai_sweep_marker_path()));
+}
+
+function vv_ai_sweep_mark(int $ts): void {
+    if (!is_dir(STATE_DIR)) @mkdir(STATE_DIR, 0755, true);
+    @file_put_contents(vv_ai_sweep_marker_path(), (string)$ts, LOCK_EX);
+}
+
+// Run records that finished after $since. A record still marked running is skipped rather than
+// read half-written — it will be picked up on the pass after it finishes.
+function vv_ai_recent_runs(int $since): array {
+    $out  = [];
+    $base = realpath(LOG_DIR);
+    if ($base === false) return [];
+
+    foreach ((array)@glob($base . '/{,*/,*/*/,*/*/*/}*.json', GLOB_BRACE) as $path) {
+        $r = json_decode((string)@file_get_contents($path), true);
+        if (!is_array($r) || empty($r['id']) || ($r['status'] ?? '') === 'running') continue;
+
+        $end = (int)($r['end'] ?? 0);
+        if ($end <= $since) continue;
+
+        $log = preg_replace('/\.json$/', '.log', $path);
+        if (!is_file($log)) continue;
+
+        $out[] = ['id' => (string)$r['id'], 'status' => (string)($r['status'] ?? '?'),
+                  'start' => (int)($r['start'] ?? 0), 'end' => $end, 'log' => $log];
+    }
+    usort($out, fn($a, $b) => $a['end'] <=> $b['end']);
+    return $out;
+}
+
+// The lines one run wrote, and only those. Logs are appended across runs, so a tail alone would
+// re-read the previous run's output and re-report faults that have already been dealt with.
+// Filtering on the leading timestamp scopes the evidence to the run being examined.
+function vv_ai_run_log_lines(string $logPath, int $startTs, int $maxLines = 2000): array {
+    $out = []; $rc = 0;
+    @exec('tail -n ' . (int)$maxLines . ' ' . escapeshellarg($logPath) . ' 2>/dev/null', $out, $rc);
+    if ($rc !== 0) return [];
+
+    $kept = [];
+    foreach ($out as $line) {
+        if (preg_match('/^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})/', $line, $m)) {
+            // A line older than the run belongs to a previous one. Two seconds of slack because
+            // the record's start is stamped by the runner, not by the first line the job writes.
+            if (strtotime($m[1]) < $startTs - 2) continue;
+        }
+        $kept[] = $line;
+    }
+    return $kept;
+}
+
+// One pass. Returns a summary rather than logging it, so the caller decides what to record and
+// the whole thing stays testable without a log to read afterwards.
+//
+// $dryRun does everything except write conf and move the marker — including probing, which is
+// the point: it answers "what would this have done" with real evidence rather than a guess.
+function vv_ai_repair_sweep(bool $dryRun = false): array {
+    if (!vv_ai_repair_enabled()) {
+        return ['ok' => false, 'error' => 'AI_REPAIR_ENABLED is not true', 'runs' => 0];
+    }
+
+    $started = time();
+    $since   = vv_ai_sweep_last();
+    $runs    = vv_ai_recent_runs($since);
+
+    $sum = ['ok' => true, 'runs' => count($runs), 'findings' => 0, 'fixed' => 0,
+            'needs_operator' => 0, 'resolved' => 0, 'quiet' => 0, 'details' => []];
+
+    foreach ($runs as $run) {
+        $lines = vv_ai_run_log_lines($run['log'], $run['start']);
+        if (!$lines) continue;
+
+        $rel = ltrim(str_replace(realpath(LOG_DIR), '', $run['log']), '/');
+        foreach (vv_ai_triage_log($lines, $rel) as $cand) {
+            $cand = vv_ai_probe_finding($cand);
+            $sum['findings']++;
+
+            // Write first, so a finding exists even if the repair below fails. A repair that
+            // errored without leaving a record is the one failure mode there is no way back from.
+            $w = vv_ai_finding_write($cand);
+            if (!($w['ok'] ?? false)) continue;
+            $id = $w['id'];
+
+            // Already acknowledged or dismissed — the operator has spoken, and re-fixing behind
+            // them would be the opposite of what an acknowledgement means.
+            if (in_array($w['state'] ?? '', ['acknowledged', 'dismissed'], true)) {
+                $sum['quiet']++;
+                continue;
+            }
+
+            if (($cand['state'] ?? '') === 'resolved') {
+                vv_ai_finding_close($id, (string)($cand['note'] ?? ''));
+                $sum['resolved']++;
+                continue;
+            }
+
+            if (vv_ai_finding_may_autofix($cand)) {
+                if ($dryRun) { $sum['details'][] = "would fix {$cand['conf_key']} → {$cand['proposed']}"; continue; }
+                $r = vv_ai_finding_apply_action($id, 'fix', 'Probed and written by the repair sweep.');
+                if ($r['ok'] ?? false) { $sum['fixed']++; $sum['details'][] = "fixed {$cand['conf_key']}"; }
+                else                   { $sum['needs_operator']++; vv_ai_finding_set_state($id, 'needs_operator', (string)($r['error'] ?? '')); }
+                continue;
+            }
+
+            if (($cand['state'] ?? '') === 'needs_operator') $sum['needs_operator']++;
+        }
+    }
+
+    // Marked only on a completed pass, and to when the pass began — a job that finished while
+    // this was running is then picked up next time instead of being skipped for having ended
+    // before a marker written at the end.
+    if (!$dryRun) vv_ai_sweep_mark($started);
+
+    return $sum;
+}
+
 // ── Answering a finding in words ─────────────────────────────────────────────────────────────
 // The buttons are unambiguous by construction. This is for the other path — replying "yeah go
 // ahead" in the chat that raised the finding — and it is matched here rather than asked of the
@@ -160,6 +160,8 @@ const VV_SCRIPT_CONF_SECTIONS = [
    'AI/ai_index.sh'                                  => ['AI Retrieval Index', 'AI Master Switch', 'Ollama'],
    'AI/ai_query.sh'                                  => ['AI Retrieval Index', 'AI Master Switch', 'Ollama'],
    'AI/ai_token_sync.sh'                             => ['AI Feature Toggles'],
+    'Plugin/unraid/Tools/ai_repair_sweep.sh'          => ['AI Repair', 'AI Repair Findings',
+                                                          'AI Master Switch'],
    // ── Shared host sections ──────────────────────────────────────────────────────────────────
    // A script's settings are not only the ones named after it. Anything talking to Lidarr reads
    // the host's Lidarr block; anything reading playback reads Emby. Those blocks are where the