diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 7f62d29..d3a345a 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -403,8 +403,13 @@ "Watchdogs/resource_watchdog.sh" # reduce system pressure before healing attempts "Watchdogs/docker_watchdog.sh" # heal containers with freed resources "Watchdogs/system_watchdog.sh" # system component health — storage + webgui + "Plugin/unraid/Tools/ai_repair_sweep.sh" # read what the last cycle logged; off unless AI_REPAIR_ENABLED "Watchdogs/stability_watchdog.sh" # reboot if all else fails — last line of defense ) +# The repair sweep sits ahead of stability deliberately, which is the one exception to "stability +# last". It reads the previous cycle's logs and may correct the very misconfiguration stability +# would otherwise reboot for — a wrong port is not fixed by restarting the machine. It is bounded +# by AI_PROBE_TIMEOUT, exits 0 in every case, and does nothing at all unless AI_REPAIR_ENABLED. # ━━━ System Watchdog ━━━ # system_watchdog.sh runs SYSTEM_WATCHDOG_SCRIPTS sequentially each cycle. diff --git a/Plugin/unraid/Tools/ai_repair_sweep.php b/Plugin/unraid/Tools/ai_repair_sweep.php new file mode 100644 index 0000000..8bf3af9 --- /dev/null +++ b/Plugin/unraid/Tools/ai_repair_sweep.php @@ -0,0 +1,103 @@ + 0 && $dryRun) rlog(sprintf('dry-run: %d run(s), nothing found (%dms)', $sum['runs'], $ms)); + exit(0); + } + + rlog(sprintf('%s%d run(s): %d finding(s), %d fixed, %d for the operator, %d resolved, %d quiet (%dms)', + $dryRun ? 'dry-run: ' : '', $sum['runs'], $sum['findings'], $sum['fixed'], + $sum['needs_operator'], $sum['resolved'], $sum['quiet'], $ms)); + + foreach ($sum['details'] as $d) rlog(' ' . $d); + +} finally { + flock($lock, LOCK_UN); + fclose($lock); +} + +exit(0); diff --git a/Plugin/unraid/Tools/ai_repair_sweep.sh b/Plugin/unraid/Tools/ai_repair_sweep.sh new file mode 100755 index 0000000..7a1d9e8 --- /dev/null +++ b/Plugin/unraid/Tools/ai_repair_sweep.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# ============================================================================================== +# ================================== AI Repair Sweep =========================================== +# ============================================================================================== +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Reads the logs of jobs that finished since the last pass, turns known error shapes into +# findings, probes for a correction, and either writes a proven value or leaves the finding +# for the operator to answer. +# Runs from the watchdog orchestrator. Off unless AI_REPAIR_ENABLED is true. +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# A one-line shim: exec php on ai_repair_sweep.php in the same directory. +# The logic is PHP because everything it needs already is — the conf writer with its backups +# and read-back verification, the findings store, and the probe layer are all functions the +# WebGUI shares. A bash reimplementation would be a second conf writer, which is precisely the +# drift the guarded write path exists to prevent. +# +# There is no post-run hook in Varaverk; nothing fires when a job finishes. The sweep picks up +# completed run records instead, so this is one entry in an orchestrator list rather than a +# call added to forty scripts. +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# A Shim, Not a Program +# This file exists only because the scheduler runs shell scripts and the work is PHP. +# Anything added here would be logic the WebGUI cannot reach, and the operator answering a +# finding in the browser must take exactly the same path as the sweep that filed it. +# +# Detecting And Repairing Are Separate Trusts +# AI_REPAIR_ENABLED alone reads logs, files findings and proposes fixes, writing nothing. +# AI_REPAIR_AUTOFIX_ENABLED is what allows a value to be written, and only ever one a probe +# has answered on. Both live in master.conf; neither is set by this script. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# Never Fatal +# Always exits 0 — on a disabled feature, a held lock, or a failed pass. The watchdog +# orchestrator runs real work either side of this, and a repair sweep must never be the +# reason a cycle reports failure. +# +# One Sweep At A Time +# The PHP takes a non-blocking flock. A pass that overruns its slot cannot have a second +# copy start probing and writing conf underneath it. +# +# Nothing Is Written That Has Not Answered +# A value reaches conf only after a probe got a response from it. Toggles are never written +# unattended at all — whether something should be switched on is a decision about intent, +# and a probe cannot prove intent. +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# AI_ENABLED master switch; nothing here runs without it +# AI_REPAIR_ENABLED read logs and file findings +# AI_REPAIR_AUTOFIX_ENABLED allow a proven value to be written unattended +# AI_PROBE_TIMEOUT seconds a single probe may take +# AI_FINDING_RETAIN_DAYS how long closed findings are kept +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# ai_repair_sweep.sh +# One pass. Files findings, applies proven fixes if autofix is on. +# +# ai_repair_sweep.sh --dry-run +# Probes and reports what it would do. Writes no conf and does not move the marker, so the +# same runs are examined again next pass. +# +# ai_repair_sweep.sh --status +# Both switches, when the last pass ran, and every open finding. +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +php "$SCRIPT_DIR/ai_repair_sweep.php" "$@" diff --git a/Plugin/unraid/include/ai_repair.php b/Plugin/unraid/include/ai_repair.php index 4dc93d9..9041857 100644 --- a/Plugin/unraid/include/ai_repair.php +++ b/Plugin/unraid/include/ai_repair.php @@ -383,6 +383,137 @@ function vv_ai_findings_for_chat(int $limit = 3): array { return array_slice(vv_ai_findings_list(['needs_operator']), 0, max(1, $limit)); } +// ── The sweep ──────────────────────────────────────────────────────────────────────────────── +// There is no post-run hook in Varaverk — nothing fires when a job finishes. Rather than add a +// call to forty scripts, this picks up run records that completed since the last pass. One entry +// in an orchestrator's list instead of forty edits, and it batches naturally. +// +// Runs that reported ok are read too. A container failing its HTTP check warns and leaves the +// watchdog exiting 0, so "only look at failures" would miss the whole class of fault this exists +// for: the job worked, and told you something is wrong. + +function vv_ai_sweep_marker_path(): string { + return STATE_DIR . '/ai_repair_sweep.db'; +} + +function vv_ai_sweep_last(): int { + return (int)trim((string)@file_get_contents(vv_ai_sweep_marker_path())); +} + +function vv_ai_sweep_mark(int $ts): void { + if (!is_dir(STATE_DIR)) @mkdir(STATE_DIR, 0755, true); + @file_put_contents(vv_ai_sweep_marker_path(), (string)$ts, LOCK_EX); +} + +// Run records that finished after $since. A record still marked running is skipped rather than +// read half-written — it will be picked up on the pass after it finishes. +function vv_ai_recent_runs(int $since): array { + $out = []; + $base = realpath(LOG_DIR); + if ($base === false) return []; + + foreach ((array)@glob($base . '/{,*/,*/*/,*/*/*/}*.json', GLOB_BRACE) as $path) { + $r = json_decode((string)@file_get_contents($path), true); + if (!is_array($r) || empty($r['id']) || ($r['status'] ?? '') === 'running') continue; + + $end = (int)($r['end'] ?? 0); + if ($end <= $since) continue; + + $log = preg_replace('/\.json$/', '.log', $path); + if (!is_file($log)) continue; + + $out[] = ['id' => (string)$r['id'], 'status' => (string)($r['status'] ?? '?'), + 'start' => (int)($r['start'] ?? 0), 'end' => $end, 'log' => $log]; + } + usort($out, fn($a, $b) => $a['end'] <=> $b['end']); + return $out; +} + +// The lines one run wrote, and only those. Logs are appended across runs, so a tail alone would +// re-read the previous run's output and re-report faults that have already been dealt with. +// Filtering on the leading timestamp scopes the evidence to the run being examined. +function vv_ai_run_log_lines(string $logPath, int $startTs, int $maxLines = 2000): array { + $out = []; $rc = 0; + @exec('tail -n ' . (int)$maxLines . ' ' . escapeshellarg($logPath) . ' 2>/dev/null', $out, $rc); + if ($rc !== 0) return []; + + $kept = []; + foreach ($out as $line) { + if (preg_match('/^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})/', $line, $m)) { + // A line older than the run belongs to a previous one. Two seconds of slack because + // the record's start is stamped by the runner, not by the first line the job writes. + if (strtotime($m[1]) < $startTs - 2) continue; + } + $kept[] = $line; + } + return $kept; +} + +// One pass. Returns a summary rather than logging it, so the caller decides what to record and +// the whole thing stays testable without a log to read afterwards. +// +// $dryRun does everything except write conf and move the marker — including probing, which is +// the point: it answers "what would this have done" with real evidence rather than a guess. +function vv_ai_repair_sweep(bool $dryRun = false): array { + if (!vv_ai_repair_enabled()) { + return ['ok' => false, 'error' => 'AI_REPAIR_ENABLED is not true', 'runs' => 0]; + } + + $started = time(); + $since = vv_ai_sweep_last(); + $runs = vv_ai_recent_runs($since); + + $sum = ['ok' => true, 'runs' => count($runs), 'findings' => 0, 'fixed' => 0, + 'needs_operator' => 0, 'resolved' => 0, 'quiet' => 0, 'details' => []]; + + foreach ($runs as $run) { + $lines = vv_ai_run_log_lines($run['log'], $run['start']); + if (!$lines) continue; + + $rel = ltrim(str_replace(realpath(LOG_DIR), '', $run['log']), '/'); + foreach (vv_ai_triage_log($lines, $rel) as $cand) { + $cand = vv_ai_probe_finding($cand); + $sum['findings']++; + + // Write first, so a finding exists even if the repair below fails. A repair that + // errored without leaving a record is the one failure mode there is no way back from. + $w = vv_ai_finding_write($cand); + if (!($w['ok'] ?? false)) continue; + $id = $w['id']; + + // Already acknowledged or dismissed — the operator has spoken, and re-fixing behind + // them would be the opposite of what an acknowledgement means. + if (in_array($w['state'] ?? '', ['acknowledged', 'dismissed'], true)) { + $sum['quiet']++; + continue; + } + + if (($cand['state'] ?? '') === 'resolved') { + vv_ai_finding_close($id, (string)($cand['note'] ?? '')); + $sum['resolved']++; + continue; + } + + if (vv_ai_finding_may_autofix($cand)) { + if ($dryRun) { $sum['details'][] = "would fix {$cand['conf_key']} → {$cand['proposed']}"; continue; } + $r = vv_ai_finding_apply_action($id, 'fix', 'Probed and written by the repair sweep.'); + if ($r['ok'] ?? false) { $sum['fixed']++; $sum['details'][] = "fixed {$cand['conf_key']}"; } + else { $sum['needs_operator']++; vv_ai_finding_set_state($id, 'needs_operator', (string)($r['error'] ?? '')); } + continue; + } + + if (($cand['state'] ?? '') === 'needs_operator') $sum['needs_operator']++; + } + } + + // Marked only on a completed pass, and to when the pass began — a job that finished while + // this was running is then picked up next time instead of being skipped for having ended + // before a marker written at the end. + if (!$dryRun) vv_ai_sweep_mark($started); + + return $sum; +} + // ── Answering a finding in words ───────────────────────────────────────────────────────────── // The buttons are unambiguous by construction. This is for the other path — replying "yeah go // ahead" in the chat that raised the finding — and it is matched here rather than asked of the diff --git a/Plugin/unraid/include/confform.php b/Plugin/unraid/include/confform.php index 1d0372f..be8521c 100644 --- a/Plugin/unraid/include/confform.php +++ b/Plugin/unraid/include/confform.php @@ -160,6 +160,8 @@ const VV_SCRIPT_CONF_SECTIONS = [ 'AI/ai_index.sh' => ['AI Retrieval Index', 'AI Master Switch', 'Ollama'], 'AI/ai_query.sh' => ['AI Retrieval Index', 'AI Master Switch', 'Ollama'], 'AI/ai_token_sync.sh' => ['AI Feature Toggles'], + 'Plugin/unraid/Tools/ai_repair_sweep.sh' => ['AI Repair', 'AI Repair Findings', + 'AI Master Switch'], // ── Shared host sections ────────────────────────────────────────────────────────────────── // A script's settings are not only the ones named after it. Anything talking to Lidarr reads // the host's Lidarr block; anything reading playback reads Emby. Those blocks are where the