diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 5cf36aa..a84946d 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -1821,7 +1821,7 @@ # Tier 1 is narration — it cannot change a decision. Tier 2 adds context to a decision a script # already made. Tier 3 assists a human. Enable in that order, and give each one weeks. AI_ASSIST_REPORTS=false # tier 1 — digest / coffee report narration - AI_ASSIST_WATCHDOG=false # tier 2 — context on a flagged condition + AI_ASSIST_WATCHDOG=false # tier 2 — file a finding when a watchdog counter passes its limit (needs AI_REPAIR_ENABLED) AI_ASSIST_DISCOVERY=false # tier 2 — discovery / classification judgement calls AI_ASSIST_CLEANUP=false # tier 2 — orphan and stuck-import triage AI_ASSIST_ONBOARD=false # tier 3 — onboarding / settings assistance diff --git a/Plugin/unraid/include/ai_repair.php b/Plugin/unraid/include/ai_repair.php index 15e502b..4523a2a 100644 --- a/Plugin/unraid/include/ai_repair.php +++ b/Plugin/unraid/include/ai_repair.php @@ -11,6 +11,11 @@ // because the repair may need something only a human can supply, and that conversation has to // survive the page being closed. // +// Four candidate sources: the arrs' own health, the system log, container logs, and the +// watchdog counters. The first three read text; the fourth reads state the watchdogs already +// maintain, and is the only one whose findings can never be repaired from here — see +// vv_ai_watchdog_findings() for why that is structural rather than a setting. +// // WHY THIS IS NOT ai_bugs // Same storage shape, different lifecycle, and the difference is the whole reason for a second // store. A bug is open until Varaverk's code changes; nothing on this host can close it. A @@ -85,6 +90,7 @@ const VV_AI_FINDING_KINDS = [ 'arr_health' => 'an arr is reporting a problem about itself', 'system_fault' => 'the kernel reported a hardware or filesystem fault about this machine', 'container_fault' => 'a container is logging a fault about its own environment', + 'watchdog_strike' => 'a watchdog has counted something far enough to be worth a record', ]; // Which kinds are a statement about Varaverk's configuration, and which are a statement about @@ -223,6 +229,18 @@ function vv_ai_repair_autofix_enabled(): bool { return strtolower(trim((string)(vv_conf_vars()['AI_REPAIR_AUTOFIX_ENABLED'] ?? 'false'))) === 'true'; } +// The first consumer AI_ASSIST_WATCHDOG has ever had. It shipped with the rest of the AI_ASSIST_ +// tier flags and nothing read it, which meant a switch that looked like it did something. +// +// Separate from AI_REPAIR_ENABLED rather than folded into it because the two answer different +// questions. Repair asks "may this read job logs and offer to change conf". This asks "may this +// have an opinion about what the watchdogs are counting" — a different subject, a different +// appetite for noise, and one you may well want off while repair stays on. +function vv_ai_assist_watchdog_enabled(): bool { + if (!vv_ai_repair_enabled()) return false; + return strtolower(trim((string)(vv_conf_vars()['AI_ASSIST_WATCHDOG'] ?? 'false'))) === 'true'; +} + function vv_ai_findings_dir(): string { $d = AI_DATA_DIR . '/ai_findings'; if (!is_dir($d)) @mkdir($d, 0755, true); @@ -808,6 +826,145 @@ function vv_ai_container_findings(int $since, ?array $logsByContainer = null): a return $found; } +// ── What the watchdogs have counted ────────────────────────────────────────────────────────── +// The fourth candidate source. The other three read logs; this reads the counters the watchdogs +// keep, and turns the ones that have gone far enough into a record with a name on it. +// +// It reads vv_wd_all() rather than the state files, for the reason the Watchdog tab does: the +// meaning of a strike lives in include/watchdog.php, and a second parser here would be a second +// opinion about it. It also gets the thresholds in the same payload, so a count is always judged +// against the limit that was in force when it was read rather than one fetched separately. +// +// Deliberately not conf-bound. 'watchdog_strike' is absent from VV_AI_CONF_BOUND_KINDS, so these +// findings carry no conf key, so vv_ai_finding_may_autofix() can never return true for one. That +// is a structural guarantee rather than a switch: there is no setting that makes this able to +// change anything, and none can be added without also deciding which key it would write. +// +// What it does NOT do is explain. The text below is derived, not generated — no model is asked +// anything on this path. The reading is a click away in the troubleshoot profile, already scoped +// to the orchestrator's log by the Why? buttons on the Watchdog tab. Detection has to be worth +// believing before it is worth narrating, and a wrong cause stated fluently is worse than none. +const VV_AI_WD_PRESSURE_LEVELS = [2 => 'medium', 3 => 'hard']; + +function vv_ai_watchdog_findings(?array $payload = null): array { + if ($payload === null && !vv_ai_assist_watchdog_enabled()) return []; + + $wd = $payload ?? vv_cache_read('watchdog', 900); + if ($wd === null) { + require_once __DIR__ . '/watchdog.php'; + $wd = vv_wd_all(); + } + + $cfg = $wd['cfg'] ?? []; + $found = []; + + foreach ($wd['nodes'] ?? [] as $node) { + // This host only. A partner's counters are its own to file findings about, and the + // operator reading them here could not act on them from this side anyway. + if (empty($node['is_me'])) continue; + + $st = $node['states'] ?? null; + if (!$st) continue; + + $id = (string)($node['id'] ?? 'this host'); + // The operator has already said they know about these. Suppressing here rather than at + // write time keeps them out of the pin store entirely — 7DaysToDie is the worked example: + // knowingly broken, deliberately ignored, and a finding about it every fifteen minutes + // would be exactly the crying-wolf this is supposed to avoid. + $ignore = array_flip((array)($node['config']['ignore'] ?? [])); + + $add = function (string $subject, string $ref, string $level, string $observed, + string $evidence) use (&$found) { + $found[] = [ + 'kind' => 'watchdog_strike', + 'subject' => $subject, + 'pin' => 'wd:' . $ref . '|' . $subject, + 'ref' => $ref, + 'conf_key' => '', + 'conf_file' => '', + 'sys_level' => $level, + 'observed' => $observed, + 'evidence' => $evidence, + // Every watchdog logs through the orchestrator; none keeps a log of its own. + 'source_log' => 'Orchestrators/watchdog_orchestrator', + 'state' => 'needs_operator', + ]; + }; + + // 1. Skip list — the terminal state. The watchdog tried its restart limit and stopped, + // which means nothing further is coming automatically. Always worth a record. + foreach ((array)($st['skiplist'] ?? []) as $name) { + if (isset($ignore[$name])) continue; + $add($name, 'skiplist', 'error', 'on the skip list', + sprintf('%s is on the docker watchdog skip list on %s. It was restarted up to ' + . 'the limit of %d and did not stay up, so the watchdog has stopped trying ' + . 'and will not restart it again without intervention.', + $name, $id, (int)($cfg['restart_limit'] ?? 3))); + } + + // 2. Container check strikes at or past their limit. Below the limit the watchdog is + // still working the problem and a finding would be premature. + $lim = max(1, (int)($cfg['cpu_fail_lim'] ?? 2)); + foreach ((array)($st['ctr_strikes'] ?? []) as $key => $count) { + $name = preg_replace('/_(cpu|http|api|docker)$/', '', (string)$key); + if (isset($ignore[$name]) || (int)$count < $lim) continue; + $check = preg_match('/_(cpu|http|api|docker)$/', (string)$key, $m) ? $m[1] : 'health'; + $add($name, 'strike:' . $check, 'warn', $count . ' / ' . $lim, + sprintf('The docker watchdog has counted %d consecutive %s failures for %s on %s, ' + . 'against a limit of %d.', (int)$count, $check, $name, $id, $lim)); + } + + // 3. Repeated restarts inside the 24h window. Distinct from a strike: the container is + // coming back each time, which is why nothing has escalated, and is also why this is + // easy to never notice. + $rc = []; + foreach ((array)($st['restarts'] ?? []) as $r) { + $n = (string)($r['name'] ?? ''); + if ($n !== '') $rc[$n] = ($rc[$n] ?? 0) + 1; + } + $rlim = max(2, (int)($cfg['restart_limit'] ?? 3)); + foreach ($rc as $name => $count) { + if (isset($ignore[$name]) || $count < $rlim) continue; + $add($name, 'restarts', 'warn', $count . ' in 24h', + sprintf('%s has been restarted %d times in the last 24 hours on %s. Each one ' + . 'succeeded, so no strike escalated — but a container that needs restarting ' + . 'this often is failing at something between restarts.', $name, $count, $id)); + } + + // 4. Any unattended reboot. Not thresholded: the machine restarting itself is worth + // saying once every time it happens, however normal the count is. + $reboots = (array)($st['reboots'] ?? []); + if ($reboots) { + $add($id, 'reboots', 'error', count($reboots) . ' / ' . (int)($cfg['reboot_limit'] ?? 3), + sprintf('The stability watchdog has rebooted %s %d time%s in the last %d hours, ' + . 'against a limit of %d before it stops trying.', + $id, count($reboots), count($reboots) === 1 ? '' : 's', + (int)($cfg['reboot_window'] ?? 12), (int)($cfg['reboot_limit'] ?? 3))); + } + + // 5. Sustained memory pressure. Level 1 is soft and self-clears constantly; from level 2 + // the watchdog has acted on containers, and that is a thing that happened. + $lvl = (int)($st['rw_level'] ?? 0); + if (isset(VV_AI_WD_PRESSURE_LEVELS[$lvl])) { + $paused = array_values(array_filter((array)($st['rw_paused'] ?? []))); + $stopped = array_values(array_filter((array)($st['rw_stopped'] ?? []))); + $acted = array_merge( + $paused ? ['paused ' . implode(', ', $paused)] : [], + $stopped ? ['stopped ' . implode(', ', $stopped)] : [] + ); + $add($id, 'pressure', $lvl >= 3 ? 'error' : 'warn', + VV_AI_WD_PRESSURE_LEVELS[$lvl] . ' pressure', + sprintf('The resource watchdog is at %s pressure on %s%s.%s', + VV_AI_WD_PRESSURE_LEVELS[$lvl], $id, + $acted ? ' and has ' . implode(' and ', $acted) : '', + !empty($st['mem_shutdown']) ? ' The memory shutdown flag is set, so ' + . 'docker_watchdog is deferring container restarts.' : '')); + } + } + + return $found; +} + // ── Reaching the operator ──────────────────────────────────────────────────────────────────── // A finding nobody is told about is a finding nobody has. The card on the AI tab shows them, but // only to someone who opens the tab, and the point of this subsystem is that it works while @@ -994,16 +1151,21 @@ function vv_ai_repair_sweep(bool $dryRun = false): array { $sum = ['ok' => true, 'runs' => count($runs), 'findings' => 0, 'fixed' => 0, 'needs_operator' => 0, 'resolved' => 0, 'quiet' => 0, 'details' => []]; - // Candidates from three sources. Log triage is bounded to runs that finished since the last + // Candidates from four sources. Log triage is bounded to runs that finished since the last // pass; the arrs are asked every time, because their health is a current state rather than // something that appeared in a log once. Asking costs three local HTTP calls. // // The system log is bounded the same way the run logs are — everything since the marker — // so a fault that has been shouting for an hour is counted once per pass rather than once // per line, and stops producing candidates the moment it stops being logged. + // + // The watchdog counters are asked every time for the same reason the arrs are: a strike count + // is a current state, not something that appeared in a log once. It is also the cheapest of + // the four — one cached file read — and gated separately, on AI_ASSIST_WATCHDOG. $candidates = vv_ai_arr_health_findings(); foreach (vv_ai_syslog_findings($since) as $c) $candidates[] = $c; foreach (vv_ai_container_findings($since) as $c) $candidates[] = $c; + foreach (vv_ai_watchdog_findings() as $c) $candidates[] = $c; foreach ($runs as $run) { $lines = vv_ai_run_log_lines($run['log'], $run['start']);