From e5edffae4b2e6ba9d66bdf23b132058428240303 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sun, 9 Aug 2026 20:17:36 -0400 Subject: [PATCH] Take the arrs at their word about their own health MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit They already publish what they believe is wrong, structured and graded, so there is nothing to parse and no severity to second-guess. Findings widen from naming a conf key to naming anything specific — Radarr's import lists being down is actionable in Radarr, not here. --- Plugin/unraid/include/ai_repair.php | 201 ++++++++++++++++++++++------ 1 file changed, 157 insertions(+), 44 deletions(-) diff --git a/Plugin/unraid/include/ai_repair.php b/Plugin/unraid/include/ai_repair.php index 9041857..a480270 100644 --- a/Plugin/unraid/include/ai_repair.php +++ b/Plugin/unraid/include/ai_repair.php @@ -22,19 +22,24 @@ // except by reading them. // // DESIGN PRINCIPLES -// A finding names a conf key or it is not a finding. +// A finding names something specific, or it is not a finding. // The point of the record is that something can be done about it. "The daily sync looked // unhappy" is a feeling; "HOST1_EMBY_URL points at a host that refuses connections" is a -// finding. The triage patterns that cannot resolve a key produce nothing rather than a -// vague row. +// finding. Triage that cannot resolve a target produces nothing rather than a vague row. +// +// For the conf-bound kinds that is a conf key, and it is still required — a proposal to +// edit a key that does not exist can never be actioned. For the rest it is whatever +// identifies the thing: an arr reporting "all lists are unavailable" is specific and +// actionable with no Varaverk key to change, because the action is in Radarr's own UI. // // Evidence is the log line, quoted. // Same rule as ai_bugs, for the same reason: a finding that cannot show the line it came // from cannot be checked, and this store is meant to be checkable. // -// Identity is kind + subject + key, not the message text. +// Identity is kind + subject + reference, not the message text. // A port that has been wrong for a week is one finding seen 400 times, not 400 findings. -// Wording drifts as logs change; the thing being wrong does not. +// Wording drifts as logs change; the thing being wrong does not. "Indexers unavailable: +// NzbNoob" becoming "NzbNoob, Miatrix" is the same finding getting worse. // // OPERATIONAL SAFEGUARDS // A proposed value is recorded, never trusted. @@ -73,8 +78,27 @@ const VV_AI_FINDING_KINDS = [ 'auth_rejected' => 'the endpoint answered, and rejected the credential', 'unknown_target' => 'a conf entry names a container or share that does not exist here', 'missing_value' => 'a conf key required by the job that ran is empty', + 'arr_health' => 'an arr is reporting a problem about itself', ]; +// Which kinds are a statement about Varaverk's configuration, and which are a statement about +// something else that is nonetheless worth recording. +// +// The rule was originally "a finding names a conf key or it is not a finding", to stop the store +// filling with observations nobody could act on. What that rule was really protecting is that +// every finding identifies something specific and actionable — naming a conf key was the proxy, +// because at the time every source of findings was a conf problem. +// +// An arr reporting "all lists are unavailable" is specific and actionable, and there is no +// Varaverk key to change: the action is in Radarr's own UI. So the identity widens to a general +// reference, and the conf-key requirement narrows to the kinds it was written for. What has not +// changed is that a finding with nothing to point at is still refused. +const VV_AI_CONF_BOUND_KINDS = ['unreachable', 'auth_rejected', 'unknown_target', 'missing_value']; + +function vv_ai_kind_is_conf_bound(string $kind): bool { + return in_array($kind, VV_AI_CONF_BOUND_KINDS, true); +} + // How a finding ended, when it ends. const VV_AI_FINDING_STATES = [ 'open' => 'seen, not yet acted on', @@ -133,7 +157,13 @@ function vv_ai_finding_severity(array $f): string { $key = (string)($f['conf_key'] ?? ''); // Deliberate-state findings never escalate, whatever their kind. - if (vv_ai_conf_is_toggle($key)) return 'warn'; + if ($key !== '' && vv_ai_conf_is_toggle($key)) return 'warn'; + + // An arr grades its own health and is the authority on it — "error" from Radarr means Radarr + // has stopped doing something, which is not a judgement to second-guess from out here. + if (($f['kind'] ?? '') === 'arr_health') { + return ($f['arr_type'] ?? '') === 'error' ? 'error' : 'warn'; + } return match ($f['kind'] ?? '') { 'auth_rejected' => 'error', // answered and refused — nothing gets through until fixed @@ -176,10 +206,15 @@ function vv_ai_finding_retain_days(): int { return max(1, $n); } -// kind + subject + conf key. Deliberately not the message: the same wrong port produces slightly +// kind + subject + reference. Deliberately not the message: the same wrong port produces slightly // different log text as the software around it changes, and that must not mint a second record. -function vv_ai_finding_id(string $kind, string $subject, string $confKey): string { - return substr(sha1(strtolower($kind . '|' . $subject . '|' . $confKey)), 0, 12); +// +// The reference is the conf key for a conf-bound kind, and whatever else identifies the thing +// otherwise — for an arr health item, the check that raised it. "Indexers unavailable: NzbNoob" +// becomes "Indexers unavailable: NzbNoob, Miatrix" as more fail, and that is the same finding +// getting worse rather than a second one. +function vv_ai_finding_id(string $kind, string $subject, string $ref): string { + return substr(sha1(strtolower($kind . '|' . $subject . '|' . $ref)), 0, 12); } function vv_ai_finding_path(string $id): ?string { @@ -204,18 +239,26 @@ function vv_ai_finding_write(array $f): array { $confKey = trim((string)($f['conf_key'] ?? '')); $evidence = trim((string)($f['evidence'] ?? '')); + // What identifies this finding. Conf-bound kinds are identified by their key; everything else + // supplies its own reference, and a finding with neither points at nothing and is refused. + $ref = trim((string)($f['ref'] ?? $confKey)); + if (!isset(VV_AI_FINDING_KINDS[$kind])) return ['ok' => false, 'error' => 'unknown kind']; - if ($subject === '' || $confKey === '') return ['ok' => false, 'error' => 'subject and conf_key required']; + if ($subject === '' || $ref === '') return ['ok' => false, 'error' => 'subject and a reference are required']; if ($evidence === '') return ['ok' => false, 'error' => 'evidence required']; - // The key has to be a real shell identifier for the same reason the conf writer insists on - // it: a finding is a proposal to edit that key, and a malformed one can never be actioned. - if (!vv_conf_key_valid($confKey)) return ['ok' => false, 'error' => 'malformed conf key']; + + if (vv_ai_kind_is_conf_bound($kind)) { + if ($confKey === '') return ['ok' => false, 'error' => 'conf_key required for this kind']; + // The key has to be a real shell identifier for the same reason the conf writer insists + // on it: a finding is a proposal to edit that key, and a malformed one cannot be actioned. + if (!vv_conf_key_valid($confKey)) return ['ok' => false, 'error' => 'malformed conf key']; + } $state = (string)($f['state'] ?? 'open'); if (!isset(VV_AI_FINDING_STATES[$state])) $state = 'open'; $now = time(); - $id = vv_ai_finding_id($kind, $subject, $confKey); + $id = vv_ai_finding_id($kind, $subject, $ref); $rec = [ 'id' => $id, 'kind' => $kind, @@ -235,7 +278,9 @@ function vv_ai_finding_write(array $f): array { 'note' => mb_substr((string)($f['note'] ?? ''), 0, 1000), // Recomputed on every sighting rather than stored once: a key that becomes a toggle, or // a toggle that is replaced by a real value, changes what this finding means. - 'severity' => vv_ai_finding_severity(['kind' => $kind, 'conf_key' => $confKey]), + 'ref' => mb_substr($ref, 0, 120), + 'severity' => vv_ai_finding_severity(['kind' => $kind, 'conf_key' => $confKey, + 'arr_type' => (string)($f['arr_type'] ?? '')]), 'host' => vv_detect_host(), 'first' => $now, 'last' => $now, @@ -466,44 +511,51 @@ function vv_ai_repair_sweep(bool $dryRun = false): array { $sum = ['ok' => true, 'runs' => count($runs), 'findings' => 0, 'fixed' => 0, 'needs_operator' => 0, 'resolved' => 0, 'quiet' => 0, 'details' => []]; + // Candidates from two sources. Log triage is bounded to runs that finished since the last + // pass; the arrs are asked every time, because their health is a current state rather than + // something that appeared in a log once. Asking costs three local HTTP calls. + $candidates = vv_ai_arr_health_findings(); + foreach ($runs as $run) { $lines = vv_ai_run_log_lines($run['log'], $run['start']); if (!$lines) continue; $rel = ltrim(str_replace(realpath(LOG_DIR), '', $run['log']), '/'); - foreach (vv_ai_triage_log($lines, $rel) as $cand) { - $cand = vv_ai_probe_finding($cand); - $sum['findings']++; + foreach (vv_ai_triage_log($lines, $rel) as $c) $candidates[] = $c; + } - // Write first, so a finding exists even if the repair below fails. A repair that - // errored without leaving a record is the one failure mode there is no way back from. - $w = vv_ai_finding_write($cand); - if (!($w['ok'] ?? false)) continue; - $id = $w['id']; + foreach ($candidates as $cand) { + $cand = vv_ai_probe_finding($cand); + $sum['findings']++; - // Already acknowledged or dismissed — the operator has spoken, and re-fixing behind - // them would be the opposite of what an acknowledgement means. - if (in_array($w['state'] ?? '', ['acknowledged', 'dismissed'], true)) { - $sum['quiet']++; - continue; - } + // Write first, so a finding exists even if the repair below fails. A repair that + // errored without leaving a record is the one failure mode there is no way back from. + $w = vv_ai_finding_write($cand); + if (!($w['ok'] ?? false)) continue; + $id = $w['id']; - if (($cand['state'] ?? '') === 'resolved') { - vv_ai_finding_close($id, (string)($cand['note'] ?? '')); - $sum['resolved']++; - continue; - } - - if (vv_ai_finding_may_autofix($cand)) { - if ($dryRun) { $sum['details'][] = "would fix {$cand['conf_key']} → {$cand['proposed']}"; continue; } - $r = vv_ai_finding_apply_action($id, 'fix', 'Probed and written by the repair sweep.'); - if ($r['ok'] ?? false) { $sum['fixed']++; $sum['details'][] = "fixed {$cand['conf_key']}"; } - else { $sum['needs_operator']++; vv_ai_finding_set_state($id, 'needs_operator', (string)($r['error'] ?? '')); } - continue; - } - - if (($cand['state'] ?? '') === 'needs_operator') $sum['needs_operator']++; + // Already acknowledged or dismissed — the operator has spoken, and re-fixing behind + // them would be the opposite of what an acknowledgement means. + if (in_array($w['state'] ?? '', ['acknowledged', 'dismissed'], true)) { + $sum['quiet']++; + continue; } + + if (($cand['state'] ?? '') === 'resolved') { + vv_ai_finding_close($id, (string)($cand['note'] ?? '')); + $sum['resolved']++; + continue; + } + + if (vv_ai_finding_may_autofix($cand)) { + if ($dryRun) { $sum['details'][] = "would fix {$cand['conf_key']} → {$cand['proposed']}"; continue; } + $r = vv_ai_finding_apply_action($id, 'fix', 'Probed and written by the repair sweep.'); + if ($r['ok'] ?? false) { $sum['fixed']++; $sum['details'][] = "fixed {$cand['conf_key']}"; } + else { $sum['needs_operator']++; vv_ai_finding_set_state($id, 'needs_operator', (string)($r['error'] ?? '')); } + continue; + } + + if (($cand['state'] ?? '') === 'needs_operator') $sum['needs_operator']++; } // Marked only on a completed pass, and to when the pass began — a job that finished while @@ -756,6 +808,67 @@ function vv_ai_triage_log(array $lines, string $sourceLog = ''): array { return array_values($found); } +// ── What the arrs say about themselves ─────────────────────────────────────────────────────── +// Sonarr, Radarr and Lidarr each publish a health endpoint listing what they believe is wrong, +// already structured and already graded. No log parsing, no pattern that goes stale when a +// message is reworded, and no guessing at severity — the arr is the authority on whether its own +// condition is an error or a warning. +// +// This is the one source here that needs no triage at all. Everything else in this file exists +// because logs are prose; these arrive as records. +// +// Note the API version differs: Lidarr is v1 where Sonarr and Radarr are v3. vv_discover_arrs() +// already carries it per arr, which is why this reads it rather than assuming. +function vv_ai_arr_health_findings(): array { + if (!function_exists('vv_discover_arrs')) { + require_once __DIR__ . '/arrs.php'; + } + + $host = vv_detect_host(); + $found = []; + + foreach (vv_discover_arrs() as $node) { + if (($node['host'] ?? '') !== $host) continue; + + foreach ($node['arrs'] ?? [] as $arr) { + $type = ucfirst((string)($arr['type'] ?? '')); + $url = (string)($arr['url'] ?? ''); + $key = (string)($arr['key'] ?? ''); + $api = (string)($arr['api'] ?? 'v3'); + if ($type === '' || $url === '' || $key === '') continue; + + $items = vv_arr_http($url, $key, "/api/$api/health", vv_ai_probe_timeout()); + // null is unreachable, which is a different finding and one the log triage already + // raises. An empty array is the arr saying it is fine, and must not be confused with + // not having been able to ask. + if (!is_array($items)) continue; + + foreach ($items as $item) { + $source = trim((string)($item['source'] ?? '')); + $message = trim((string)($item['message'] ?? '')); + if ($source === '' || $message === '') continue; + + $found[] = [ + 'kind' => 'arr_health', + 'subject' => $type, + 'ref' => $source, + 'conf_key' => '', + 'conf_file' => '', + 'arr_type' => strtolower((string)($item['type'] ?? 'warning')), + 'observed' => $message, + 'evidence' => $type . ' › ' . $source . ': ' . $message, + 'source_log' => $type . ' /api/' . $api . '/health', + // The arr's own documentation for this check, which is the actual next step + // for most of them and costs nothing to carry. + 'note' => trim((string)($item['wikiUrl'] ?? '')), + ]; + } + } + break; + } + return $found; +} + // ── Proving a candidate ────────────────────────────────────────────────────────────────────── // The guard the whole unattended path rests on: nothing is written that has not answered. //