Gmer4Lfe
eab5cc911c
Mesh is a mode rather than an AI profile — it has no model, tokens, reasoning or sources, so making it a profile would branch every profile-aware path on the one that has no model.
1095 lines
56 KiB
PHP
1095 lines
56 KiB
PHP
<?php
|
||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||
// PURPOSE
|
||
// Watchdog tab. Shows what every watchdog has recorded across all nodes — strike counts,
|
||
// action levels, paused and stopped containers, restart history, OOM and reboot counts.
|
||
//
|
||
// DESIGN PRINCIPLES
|
||
// State is a pure view. Everything the watchdogs have recorded arrives from one poll against
|
||
// api/watchdog.php, and all parsing lives in include/watchdog.php behind that endpoint, so the
|
||
// page cannot hold a second opinion about what a strike means.
|
||
//
|
||
// Configuration is not, and reads conf directly. The container lists and the settings card are
|
||
// rendered server-side from host*.conf at page load rather than from the polled payload: that
|
||
// payload is cached for up to 300s and drops the per-entry comments, which are where the reason
|
||
// for every number in these lists is written down.
|
||
//
|
||
// Shows every node, not just this one. A watchdog page that only covered the host you
|
||
// happened to open it on would miss exactly the node in trouble. Editing is local-host only
|
||
// though — the partner owns its own host*.conf.
|
||
//
|
||
// OPERATIONAL SAFEGUARDS
|
||
// Read-only about state. There is no control on this page that clears a strike, restarts a
|
||
// container, or resets a counter — those belong to the watchdog that owns the state. Changing
|
||
// which containers a watchdog acts on is a conf edit and goes through the guarded write path
|
||
// with everyone else's; it is not a way to reach into what a watchdog has already decided.
|
||
//
|
||
// That holds for the assistant too, and is why the Why? buttons open troubleshoot rather than
|
||
// repair. troubleshoot may read the machine and the orchestrator's log; it holds no conf_write
|
||
// and no way to act. A page whose stated contract is "changes nothing" must not carry the one
|
||
// profile that can — see include/ai_profiles.php for why that grant is deliberately narrow.
|
||
//
|
||
// A node that fails to report renders as unavailable rather than healthy. Absence of data
|
||
// is never drawn as an all-clear.
|
||
//
|
||
// RENDERS
|
||
// Per-node cards: resource / docker / system / storage / network watchdog state,
|
||
// strike pills, restart history, reboot and OOM counts
|
||
// Editable container lists for this host: Tier 1 memory limits, the two pressure lists, Required
|
||
// Assistant card, scoped to the Watchdog tab
|
||
// Watchdog conf sections, via the shared settings renderer
|
||
//
|
||
// DEPENDS ON
|
||
// api/watchdog.php polled every 30s → include/watchdog.php (served from a 300s cache)
|
||
// api/confform.php POST → the guarded conf write path, for the list editors
|
||
// include/ai_chat.php → the shared chat component
|
||
require_once dirname(__DIR__) . '/include/confui.php';
|
||
require_once dirname(__DIR__) . '/include/ai_chat.php';
|
||
// For vv_wd_installed_containers() only. The state on this page still arrives entirely through
|
||
// api/watchdog.php — this include buys one function, not a second collection path.
|
||
require_once dirname(__DIR__) . '/include/watchdog.php';
|
||
?>
|
||
<style>
|
||
.vv-wd-card { background:#161616;border:1px solid #2a2a2a;border-radius:6px;padding:10px;min-width:0; }
|
||
.vv-wd-sec { font-size:10px;font-weight:bold;color:#444;letter-spacing:.07em;text-transform:uppercase;margin-bottom:6px; }
|
||
.vv-wd-row { display:flex;justify-content:space-between;align-items:baseline;gap:6px;margin:2px 0; }
|
||
.vv-wd-lbl { font-size:11px;color:#444;white-space:nowrap; }
|
||
.vv-wd-val { font-size:12px;color:#bbb;text-align:right; }
|
||
.vv-wd-sep { border:none;border-top:1px solid #1e1e1e;margin:6px 0; }
|
||
.vv-wd-pill { font-size:10px;padding:1px 6px;border-radius:2px;background:#1e1e1e;color:#666;border:1px solid #272727; }
|
||
/* Unraid's default-base.css carries `span.warn { background:var(--yellow-200); display:block;
|
||
width:100% }`. The state modifier here is built at runtime as `vv-wd-pill ${cls}`, so a pill in
|
||
the warn state matched it: `.vv-wd-pill.warn` outranks it on background and colour but declares
|
||
no display or width, and the pill then filled its whole flex row and pushed the others onto the
|
||
next line. Element-qualified, so this ties on specificity and wins on order — the page's style
|
||
block is parsed after the theme's. */
|
||
span.vv-wd-pill { display:inline-block; width:auto; }
|
||
.vv-wd-pill.ok { background:#0d1f0d;color:#4caf50;border-color:#1a3a1a; }
|
||
.vv-wd-pill.warn { background:#1f1500;color:#ffb74d;border-color:#3a2800; }
|
||
.vv-wd-pill.err { background:#200d0d;color:#ef5350;border-color:#3a1a1a; }
|
||
.vv-wd-pill-row { display:flex;flex-wrap:wrap;gap:4px;margin-top:4px; }
|
||
.vv-wd-bar { height:5px;border-radius:2px;background:#1e1e1e;margin-top:3px;overflow:hidden; }
|
||
.vv-wd-bar-fill{ height:100%;border-radius:2px;transition:width .3s; }
|
||
.vv-wd-badge { font-size:11px;font-weight:bold;padding:2px 8px;border-radius:3px; }
|
||
.vv-wd-badge.ok { background:#0d1f0d;color:#4caf50; }
|
||
.vv-wd-badge.soft { background:#1f1f00;color:#cddc39; }
|
||
.vv-wd-badge.med { background:#1f1000;color:#ffb74d; }
|
||
.vv-wd-badge.hard { background:#200d0d;color:#ef5350; }
|
||
.vv-wd-node-h { display:flex;align-items:center;gap:8px;margin-bottom:10px; }
|
||
.vv-wd-node-id { font-size:12px;font-weight:bold;color:#666;letter-spacing:.06em;text-transform:uppercase; }
|
||
.vv-wd-dot { width:6px;height:6px;border-radius:50%;flex-shrink:0; }
|
||
.vv-wd-strike-row { display:flex;align-items:baseline;gap:6px;margin:2px 0; }
|
||
.vv-wd-strike-name{ font-size:11px;color:#777;flex:1; }
|
||
.vv-wd-strike-cnt { font-size:11px;font-weight:bold;color:#ffb74d; }
|
||
.vv-wd-reboot-ts { font-size:11px;color:#555;margin:1px 0; }
|
||
.vv-wd-ctr-row { display:flex;justify-content:space-between;align-items:baseline;margin:2px 0; }
|
||
.vv-wd-ctr-name{ font-size:11px;color:#888; }
|
||
.vv-wd-ctr-lim { font-size:11px;color:#555; }
|
||
.vv-wd-pressure{ grid-column:1/-1;border-color:#3a2000;background:#1a1000; }
|
||
|
||
/* ── Editable container lists ────────────────────────────────────────────────
|
||
Rows, not a textarea. The same values are reachable through the settings card below as raw
|
||
bash array bodies, which is fine for reading and unforgiving for changing one number. */
|
||
.vv-wl-card { background:#161616;border:1px solid #2a2a2a;border-radius:6px;padding:10px;min-width:0;
|
||
display:flex;flex-direction:column; }
|
||
.vv-wl-h { display:flex;align-items:baseline;gap:8px;margin-bottom:2px; }
|
||
.vv-wl-t { font-size:10px;font-weight:bold;color:#666;letter-spacing:.07em;text-transform:uppercase; }
|
||
.vv-wl-n { font-size:10px;color:#3a3a3a;margin-left:auto; }
|
||
.vv-wl-hint { font-size:10px;color:#3f3f3f;line-height:1.45;margin-bottom:8px; }
|
||
.vv-wl-row { display:flex;align-items:center;gap:5px;margin:2px 0; }
|
||
.vv-wl-row.gone .vv-wl-name { text-decoration:line-through;opacity:.4; }
|
||
.vv-wl-name { flex:1;min-width:0;background:#0d0d0d;border:1px solid #242424;color:#bbb;
|
||
font-size:11px;padding:3px 6px;border-radius:3px;font-family:inherit; }
|
||
.vv-wl-val { width:74px;flex-shrink:0;background:#0d0d0d;border:1px solid #242424;color:#ddd;
|
||
font-size:11px;padding:3px 6px;border-radius:3px;text-align:right; }
|
||
.vv-wl-unit { font-size:10px;color:#3a3a3a;flex-shrink:0;width:20px; }
|
||
.vv-wl-gb { font-size:10px;color:#4a4a4a;flex-shrink:0;width:48px;text-align:right; }
|
||
.vv-wl-x { background:none;border:none;color:#4a4a4a;cursor:pointer;font-size:13px;
|
||
line-height:1;padding:0 3px;flex-shrink:0; }
|
||
.vv-wl-x:hover{ color:#ef5350; }
|
||
.vv-wl-row.gone .vv-wl-x { color:#ef5350; }
|
||
.vv-wl-note { font-size:10px;color:#3a3a3a;margin:0 0 4px 6px;line-height:1.4; }
|
||
.vv-wl-foot { display:flex;align-items:center;gap:6px;margin-top:auto;padding-top:8px; }
|
||
.vv-wl-sum { font-size:10px;color:#3a3a3a;flex:1;min-width:0; }
|
||
.vv-wl-sum.ok { color:#4caf50; }
|
||
.vv-wl-sum.err{ color:#ef5350; }
|
||
.vv-wl-dirty { border-color:#3a4a6a !important; }
|
||
/* An entry naming a container that is not installed. Dashed rather than red: it is not an error,
|
||
it is a line that does nothing — and on the pressure lists that is the difference between what
|
||
the card implies can be shed and what actually can be. */
|
||
.vv-wl-gone-t { font-size:9px;color:#7a5020;background:#1a1000;border:1px dashed #3a2800;
|
||
border-radius:2px;padding:0 4px;flex-shrink:0;white-space:nowrap; }
|
||
.vv-wd-pill.missing { opacity:.5;border-style:dashed;color:#7a5020;border-color:#3a2800; }
|
||
.vv-wd-ctr-row.missing .vv-wd-ctr-name { opacity:.5; }
|
||
|
||
/* One host per row — inner grid sizes all cards equally */
|
||
.vv-wd-host-row {
|
||
grid-column: 1 / -1;
|
||
display: grid;
|
||
grid-template-columns: repeat(5, 1fr);
|
||
gap: 12px;
|
||
}
|
||
@media (max-width: 900px) {
|
||
.vv-wd-host-row { grid-template-columns: 1fr; }
|
||
}
|
||
</style>
|
||
|
||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;padding:0 2px;">
|
||
<span style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;">Watchdog</span>
|
||
<span style="font-size:11px;color:#3a3a3a;" id="vv-wd-ts"></span>
|
||
</div>
|
||
|
||
<div id="vv-wd-grid" style="display:grid;grid-template-columns:repeat(8,1fr);gap:12px;">
|
||
<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">Loading…</div>
|
||
</div>
|
||
|
||
|
||
<?php
|
||
// ── Editable container lists ─────────────────────────────────────────────────────────────────
|
||
// Read from conf at render time rather than from the watchdog payload, for two reasons. The
|
||
// payload is cached for up to 300s, so a list edited here would keep showing its old contents
|
||
// until the writer's next pass; and it drops the per-entry comments, which is where the reason for
|
||
// every one of these numbers is written down — "24GB, raised 2026-07-11, was creeping toward the
|
||
// ceiling" is the whole value of the row.
|
||
//
|
||
// Local host only. These live in host*.conf, and the partner owns its own: writing HOST2's limits
|
||
// from HOST1 produces a change on the wrong side of a conf sync that is currently switched off
|
||
// anyway. The remote node's lists stay visible in the inventory card below, read-only.
|
||
$_vv_wd_me = strtoupper(vv_detect_host());
|
||
$_vv_wd_file = strtolower($_vv_wd_me) . '.conf';
|
||
$_vv_wd_defs = [
|
||
// First, because it is the one people come to this page looking for: the answer to "stop
|
||
// telling me about this container". Everything below tunes what a watchdog does; this is how
|
||
// you take something out of its hands entirely.
|
||
// 'check' — whether a name that matches no installed container is worth marking. True where a
|
||
// dead entry silently costs something: a Tier 1 limit that guards nothing, or a stop-list slot
|
||
// that sheds no memory at hard pressure. False here, because ignoring something that no longer
|
||
// exists is exactly what this list is for.
|
||
['key' => $_vv_wd_me . '_WATCHDOG_SCAN_IGNORE', 'kind' => 'list', 'check' => false,
|
||
'title' => 'Scan ignore', 'unit' => '',
|
||
'hint' => 'Left alone completely — no strikes, no restarts, no unhealthy or OOM reports. '
|
||
. 'For containers you knowingly run broken or stopped. It silences real faults as '
|
||
. 'readily as noise, so it is the wrong tool for one that is merely loud.'],
|
||
['key' => $_vv_wd_me . '_WATCHDOG_CONTAINERS', 'kind' => 'map', 'check' => true,
|
||
'title' => 'Memory limits (Tier 1)', 'unit' => 'MB',
|
||
'hint' => 'Restarted when it holds more than this. Tier 1 is per container and acts before '
|
||
. 'any pressure tier does.'],
|
||
['key' => $_vv_wd_me . '_RW_PAUSE_CONTAINERS', 'kind' => 'list', 'check' => true,
|
||
'title' => 'Pause at medium pressure', 'unit' => '',
|
||
'hint' => 'Paused — not stopped — while RAM sits below the medium threshold, and resumed on '
|
||
. 'recovery. Safe for anything that can lose a few minutes.'],
|
||
['key' => $_vv_wd_me . '_RW_STOP_CONTAINERS', 'kind' => 'list', 'check' => true,
|
||
'title' => 'Stop at hard pressure', 'unit' => '',
|
||
'hint' => 'Stopped outright at the hard threshold, and not restarted automatically. The last '
|
||
. 'step before the memory shutdown.'],
|
||
['key' => $_vv_wd_me . '_WATCHDOG_REQUIRED_CONTAINERS', 'kind' => 'list', 'check' => true,
|
||
'title' => 'Required', 'unit' => '',
|
||
'hint' => 'Must be running. Missing is a strike whether or not the container is being '
|
||
. 'watched for anything else.'],
|
||
];
|
||
$_vv_wd_fields = [];
|
||
foreach (vv_conf_all_groups($_vv_wd_file) as $_g) {
|
||
foreach ($_g['fields'] as $_f) $_vv_wd_fields[$_f['key']] = $_f;
|
||
}
|
||
$_vv_wd_lists = [];
|
||
foreach ($_vv_wd_defs as $_d) {
|
||
$_f = $_vv_wd_fields[$_d['key']] ?? null;
|
||
if (!$_f) continue; // key absent from this host's conf — draw nothing
|
||
$_vv_wd_lists[] = $_d + [
|
||
'file' => $_f['file'] ?? $_vv_wd_file,
|
||
'type' => $_f['type'], // assoc_array | array — the writer splices on this
|
||
'body' => $_f['value'],
|
||
];
|
||
}
|
||
?>
|
||
<?php
|
||
// Which container names still refer to something. Emitted once and read by both surfaces that
|
||
// draw these lists — the editors below and the inventory card in the polled grid — so the two
|
||
// cannot disagree about whether an entry is dead.
|
||
//
|
||
// Empty means "could not ask docker", not "nothing is installed", and every consumer checks the
|
||
// length before badging anything. A docker outage must not repaint the page as though every list
|
||
// were full of ghosts.
|
||
$_vv_wd_installed = vv_wd_installed_containers();
|
||
?>
|
||
<script>window.VvWdInstalled = <?= json_encode($_vv_wd_installed, JSON_UNESCAPED_SLASHES) ?>;</script>
|
||
<?php if ($_vv_wd_lists): ?>
|
||
<div id="vv-wd-lists" style="display:grid;grid-template-columns:repeat(auto-fit,minmax(320px,1fr));gap:12px;margin-top:12px;"></div>
|
||
<script>window.VvWdLists = <?= json_encode($_vv_wd_lists, JSON_UNESCAPED_SLASHES) ?>;</script>
|
||
<?php endif; ?>
|
||
|
||
<?php if (vv_ai_ui_on()): ?>
|
||
<div class="vv-card" id="vv-wd-ai-card" style="margin-top:12px;">
|
||
<?php
|
||
// All three, in this order, before the markup. vv_ai_chat_markup() emits only the boxes — the
|
||
// factory that brings them to life, the profile registry the picker draws from, and the store
|
||
// that keeps a thread across a reload are separate and none of them is implied by the others.
|
||
// Omitting them renders a chat that looks complete and dies on the first click with VvAiChat
|
||
// undefined, which is how the Scheduler's panel behaved before it was given the same three.
|
||
// Each is guarded internally, so calling them here costs nothing on a page that already has.
|
||
vv_ai_profiles_script();
|
||
vv_ai_chat_store_script();
|
||
vv_ai_chat_assets();
|
||
// Reads the page rather than sitting beside it. The scope is sent with every question, and the
|
||
// worker turns it into "they have Watchdog open — read bare references against it", so "why did
|
||
// it restart that" resolves here instead of asking which of forty containers is meant.
|
||
//
|
||
// Starts on the Varaverk assistant, not General Chat as the Monitor card does. The two pages ask
|
||
// different things: the dashboard is where an idle question gets typed, this is a page you only
|
||
// open because something struck. Answers about strikes and thresholds should come from this
|
||
// installation's own docs and state, with sources, rather than from the model's general
|
||
// impression of what a watchdog usually does.
|
||
//
|
||
// No 'height' triple — this card is in normal page flow, not a fixed panel, so the component's
|
||
// own defaults apply and the expand control still works against them. The Scheduler is the only
|
||
// placement whose size is a share of something and therefore has to be told.
|
||
vv_ai_chat_markup('vv-wd-ai', [
|
||
'mesh' => true,
|
||
'profile' => 'varaverk',
|
||
'compact' => true,
|
||
'title' => 'Assistant',
|
||
'scopeLabel' => 'Watchdog',
|
||
'empty' => 'Ask about a strike, a restart, or a threshold on this page. '
|
||
. 'Why? on any strike asks against the orchestrator\'s log.',
|
||
'placeholder' => 'Ask about what is on this page…',
|
||
]); ?>
|
||
</div>
|
||
<?php endif; ?>
|
||
|
||
<?php
|
||
// The conf sections this page is about, drawn by the shared renderer. They were reachable
|
||
// only from the Settings tab's catch-all, which is a long way to go for a setting named
|
||
// after the page you are already looking at.
|
||
vv_conf_ui_card('vv-cf-watchdog', 'watchdog', 'Watchdog settings');
|
||
|
||
// The same AI switches the AI tab carries, on the page they act on. One conf value behind both,
|
||
// written through the same guarded path, so the two cards cannot disagree — a toggle flipped here
|
||
// is flipped there because there is only one of it.
|
||
vv_conf_ui_card('vv-cf-wd-ai', 'AI Feature Toggles', 'AI features');
|
||
|
||
// Findings this page's watchdogs produced, actionable here rather than only on the AI tab.
|
||
vv_ai_findings_strip('vv-wd-fnd', ['watchdog_strike'], 'Watchdog findings');
|
||
?>
|
||
<script>
|
||
(function() {
|
||
|
||
const GB = 1073741824;
|
||
|
||
// The assistant instance, once the card is on the page; null whenever AI is off. Every Why?
|
||
// button checks it, and none of them render without it — a control that opens a panel that does
|
||
// not exist is worse than no control.
|
||
let vvWdChat = null;
|
||
// What the chat says the operator is looking at. Starts as the tab itself and moves to the
|
||
// orchestrator's log when a Why? is pressed, because that is the only place any of this is
|
||
// written down: the watchdogs log through watchdog_orchestrator.sh and have no log files of
|
||
// their own — checked against /var/log/varaverk, where Watchdogs/ does not exist.
|
||
let vvWdScope = 'Watchdog';
|
||
const WD_LOG = 'Orchestrators/watchdog_orchestrator';
|
||
|
||
// Rendered as data attributes and read back by one delegated listener rather than written into an
|
||
// onclick. Container names and log paths reach these buttons from conf and from the filesystem, and
|
||
// a name containing a quote closes the attribute early and takes the rest of the handler with it.
|
||
// vvEscAttr is the one that escapes quotes; vvEscHtml deliberately does not.
|
||
function _why(label, q) {
|
||
if (!vvWdChat) return '';
|
||
return `<button class="vv-btn-sm vv-why-btn" title="Ask the assistant about this"
|
||
data-label="${vvEscAttr(label)}" data-q="${vvEscAttr(q)}">Why?</button>`;
|
||
}
|
||
|
||
function vvWdWhy(label, q) {
|
||
if (!vvWdChat || vvWdChat.busy()) return;
|
||
// Troubleshoot, never repair. This page changes nothing, and repair is the one profile that can
|
||
// — see the safeguard note in the header.
|
||
vvWdScope = WD_LOG;
|
||
vvWdChat.retarget('troubleshoot', label, 'now looking at ' + label);
|
||
const input = document.getElementById('vv-wd-ai-input');
|
||
if (input) input.value = q;
|
||
vvWdChat.send();
|
||
const card = document.getElementById('vv-wd-ai-card');
|
||
if (card) card.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
|
||
}
|
||
|
||
function _fmtBytes(b) {
|
||
if (!b) return '0';
|
||
if (b >= GB) return (b / GB).toFixed(1) + 'G';
|
||
if (b >= 1048576) return (b / 1048576).toFixed(0) + 'M';
|
||
return (b / 1024).toFixed(0) + 'K';
|
||
}
|
||
|
||
function _relTime(ts) {
|
||
if (!ts) return '—';
|
||
const s = Math.floor(Date.now() / 1000) - ts;
|
||
if (s < 60) return 'just now';
|
||
if (s < 3600) return Math.floor(s / 60) + 'm ago';
|
||
if (s < 86400) return Math.floor(s / 3600) + 'h ago';
|
||
return Math.floor(s / 86400) + 'd ago';
|
||
}
|
||
|
||
function _dur(s) {
|
||
const d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60);
|
||
if (d) return d + 'd ' + h + 'h';
|
||
if (h) return h + 'h ' + m + 'm';
|
||
return m + 'm';
|
||
}
|
||
|
||
function _row(lbl, val) {
|
||
return `<div class="vv-wd-row"><span class="vv-wd-lbl">${lbl}</span><span class="vv-wd-val">${val}</span></div>`;
|
||
}
|
||
|
||
function _bar(pct, col) {
|
||
return `<div class="vv-wd-bar"><div class="vv-wd-bar-fill" style="width:${Math.min(pct,100)}%;background:${col}"></div></div>`;
|
||
}
|
||
|
||
// Escapes its own label, so every caller passes plain text and none of them has to remember. The
|
||
// labels here are container names, log paths and DDNS domains — all of them values this page reads
|
||
// rather than writes, which is exactly the category that must not be trusted into markup.
|
||
function _pill(label, cls) {
|
||
return `<span class="vv-wd-pill ${cls}">${vvEscHtml(label)}</span>`;
|
||
}
|
||
|
||
// ── Does this name still refer to a container? ───────────────────────────────
|
||
// One resolver for both surfaces. Answers false only when it is certain: an empty install set
|
||
// means docker could not be asked, and every caller then badges nothing rather than everything.
|
||
const _installed = new Set(window.VvWdInstalled || []);
|
||
function _known(name) { return !_installed.size || _installed.has(String(name || '').trim()); }
|
||
|
||
// Applied to the lists whose entries are container names, and deliberately NOT to the appdata
|
||
// ceilings. Those are documented as containers but hold `7dtd`, which is an appdata folder — the
|
||
// container is 7DaysToDie. Until that rule is pinned down, badging them would mark a working
|
||
// entry dead, and a wrong badge on this page is worse than a missing one: the whole value of it
|
||
// is that it is believed.
|
||
function _pillK(label, cls) {
|
||
return _known(label) ? _pill(label, cls)
|
||
: `<span class="vv-wd-pill ${cls} missing" title="No container by this name is installed — this entry does nothing">${vvEscHtml(label)}</span>`;
|
||
}
|
||
|
||
function _levelLabel(level) {
|
||
return ['OK', 'SOFT', 'MEDIUM', 'HARD'][level] || '?';
|
||
}
|
||
function _levelCls(level) {
|
||
return ['ok', 'soft', 'med', 'hard'][level] || 'ok';
|
||
}
|
||
|
||
// ── Pressure alert card ───────────────────────────────────────────────────────
|
||
function _pressureCard(node) {
|
||
const st = node.states;
|
||
if (!st || st.rw_level === 0) return '';
|
||
const level = st.rw_level;
|
||
const cls = _levelCls(level);
|
||
const label = _levelLabel(level);
|
||
|
||
const paused = (st.rw_paused || []).filter(Boolean);
|
||
const stopped = (st.rw_stopped || []).filter(Boolean);
|
||
const pausedHtml = paused.length ? paused.map(c => _pill(c, 'warn')).join('') : '';
|
||
const stoppedHtml = stopped.length ? stopped.map(c => _pill(c, 'err')).join('') : '';
|
||
|
||
return `<div class="vv-wd-card vv-wd-pressure" style="grid-column:1/-1">
|
||
<div style="display:flex;align-items:center;gap:10px;margin-bottom:8px;">
|
||
<span class="vv-wd-badge ${cls}">PRESSURE ${label}</span>
|
||
<span style="font-size:11px;color:#7a5020;">${vvEscHtml(node.id)} (${vvEscHtml(node.hostname)})</span>
|
||
${_why(node.id + ' memory pressure',
|
||
'The resource watchdog is at ' + label + ' pressure on ' + node.id
|
||
+ '. What drove it there, and what did it pause or stop?')}
|
||
${st.mem_shutdown ? `<span class="vv-wd-badge hard" style="margin-left:auto;">MEM SHUTDOWN ACTIVE</span>` : ''}
|
||
</div>
|
||
${paused.length ? `<div style="margin-bottom:4px;"><span class="vv-wd-lbl">Paused:</span> <span class="vv-wd-pill-row" style="display:inline-flex;">${pausedHtml}</span></div>` : ''}
|
||
${stopped.length ? `<div><span class="vv-wd-lbl">Stopped:</span> <span class="vv-wd-pill-row" style="display:inline-flex;">${stoppedHtml}</span></div>` : ''}
|
||
</div>`;
|
||
}
|
||
|
||
// ── System health card ────────────────────────────────────────────────────────
|
||
function _systemCard(node, cfg) {
|
||
const sys = node.system;
|
||
if (!sys) {
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-node-h">
|
||
<span class="vv-wd-dot" style="background:#444"></span>
|
||
<span class="vv-wd-node-id">${vvEscHtml(node.id)}</span>
|
||
<span style="font-size:11px;color:#3a3a3a;">${vvEscHtml(node.hostname)}</span>
|
||
<span class="vv-wd-badge" style="margin-left:auto;background:#1a1a1a;color:#444">UNREACHABLE</span>
|
||
</div>
|
||
</div>`;
|
||
}
|
||
|
||
const memGb = sys.mem_avail / GB;
|
||
const memTotGb = sys.mem_total / GB;
|
||
const usedPct = memTotGb > 0 ? ((memTotGb - memGb) / memTotGb) * 100 : 0;
|
||
const memCol = memGb < cfg.sys_mem_gb ? '#ef5350'
|
||
: memGb < cfg.rw_hard_gb ? '#ef5350'
|
||
: memGb < cfg.rw_medium_gb ? '#ffb74d'
|
||
: memGb < cfg.rw_soft_gb ? '#cddc39' : '#4caf50';
|
||
|
||
const loadPct = sys.cores > 0 ? (sys.load1 / (sys.cores * cfg.rw_load_med)) * 100 : 0;
|
||
const loadCol = sys.load1 > sys.cores * cfg.rw_load_med ? '#ef5350'
|
||
: sys.load1 > sys.cores * cfg.rw_load_soft ? '#ffb74d'
|
||
: '#4caf50';
|
||
|
||
// The fourth member of SYSTEM_WATCHDOG_SCRIPTS, and the only one with nothing to show: it keeps
|
||
// the persistent backup of the partner's conf current, and writes no state until it does. Drawn
|
||
// as its gate rather than left off the card, because a chain member that is simply missing from
|
||
// the page reads as one that is fine — the same reason an unreachable node here says so instead
|
||
// of rendering healthy. Names the switch, so "why is it off" does not need a hunt through conf.
|
||
const confCacheHtml = !cfg.fallback_on
|
||
? '<span style="color:#555;">off — FALLBACK_ENABLED</span>'
|
||
: !cfg.conf_sync_on
|
||
? '<span style="color:#7a6a20;">off — CONF_SYNC_ENABLED</span>'
|
||
: '<span style="color:#4caf50;">active</span>';
|
||
|
||
// ── Is anything still running? ──────────────────────────────────────────────
|
||
// Every other number on this card is only meaningful if a cycle actually happened. If the
|
||
// orchestrator stopped, the strike maps stop changing and the whole page renders zeros, "all
|
||
// clear" pills and green dots — its healthiest possible appearance, describing a machine with
|
||
// no watchdogs at all. This is the one reading that distinguishes quiet from absent, and the
|
||
// page's own contract says absence must never be drawn as an all-clear.
|
||
//
|
||
// stability_watchdog.sh writes the heartbeat last in the chain, so a fresh value means a whole
|
||
// cycle finished rather than one that started and died halfway. Overdue is measured against the
|
||
// schedule in force (cfg.cycle_min, read from the cron), not a constant: one missed cycle is
|
||
// worth noticing, two is worth alarming about.
|
||
const cycleMin = cfg.cycle_min || 15;
|
||
const lastCyc = (node.states || {}).last_cycle || 0;
|
||
const cycAge = lastCyc ? Math.max(0, Math.floor(Date.now() / 1000) - lastCyc) : null;
|
||
const cycCol = cycAge == null ? '#555'
|
||
: cycAge > cycleMin * 60 * 3 ? '#ef5350'
|
||
: cycAge > cycleMin * 60 * 2 ? '#ffb74d' : '#4caf50';
|
||
const cycTxt = cycAge == null ? 'never' : _relTime(lastCyc);
|
||
const cycNote = cycAge == null ? ' — no cycle has completed'
|
||
: cycAge > cycleMin * 60 * 2 ? ' — overdue, expected every ' + cycleMin + 'm' : '';
|
||
|
||
const apiOnly = sys.api_only === true;
|
||
const daemonDot = sys.daemon_ok === null ? '#555' : sys.daemon_ok ? '#4caf50' : '#ef5350';
|
||
const daemonTxt = sys.daemon_ok === null ? '—' : sys.daemon_ok ? 'daemon ok' : 'daemon err';
|
||
|
||
const st = node.states || {};
|
||
const level = st.rw_level || 0;
|
||
const dotCol = level >= 3 ? '#ef5350' : level >= 2 ? '#ffb74d' : level >= 1 ? '#cddc39'
|
||
: apiOnly ? '#4a7a9b' // blue-grey: API-only, no watchdog state
|
||
: '#4caf50';
|
||
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-node-h">
|
||
<span class="vv-wd-dot" style="background:${dotCol}"></span>
|
||
<span class="vv-wd-node-id">${vvEscHtml(node.id)}</span>
|
||
<span style="font-size:11px;color:#3a3a3a;">${vvEscHtml(node.hostname)}</span>
|
||
<span class="vv-wd-badge ${apiOnly ? '' : _levelCls(level)}"
|
||
style="margin-left:auto;${apiOnly ? 'background:#0d1f2a;color:#4a9eff;' : ''}"
|
||
>${apiOnly ? 'API ONLY' : _levelLabel(level)}</span>
|
||
</div>
|
||
<div class="vv-wd-sec">System</div>
|
||
${!apiOnly ? _row('Last cycle', `<span style="color:${cycCol}">${vvEscHtml(cycTxt)}</span>`
|
||
+ `<span style="color:#3a3a3a;font-size:10px;">${vvEscHtml(cycNote)}</span>`) : ''}
|
||
${_row('RAM free', `<span style="color:${memCol}">${_fmtBytes(sys.mem_avail)}</span> / ${_fmtBytes(sys.mem_total)}`)}
|
||
${_bar(usedPct, memCol)}
|
||
<div style="display:flex;justify-content:space-between;margin-top:1px;font-size:10px;color:#333">
|
||
<span>free</span>
|
||
<span>${cfg.rw_soft_gb}G soft · ${cfg.rw_hard_gb}G hard · ${cfg.sys_mem_gb}G reboot</span>
|
||
</div>
|
||
<div style="height:5px"></div>
|
||
${!apiOnly ? `${_row('Load avg', `<span style="color:${loadCol}">${sys.load1.toFixed(2)}</span> / ${sys.cores} cores`)}
|
||
${_bar(loadPct, loadCol)}
|
||
<div style="height:5px"></div>` : ''}
|
||
${_row('Uptime', sys.uptime ? _dur(sys.uptime) : '—')}
|
||
${_row('Docker', `<span style="color:${daemonDot}">${daemonTxt}</span>`)}
|
||
${_row('Conf cache', confCacheHtml)}
|
||
${!apiOnly ? (sys.oom_count > 0 ? _row('OOM kills', `<span style="color:#ef5350">${sys.oom_count}</span>`) : _row('OOM kills', '<span style="color:#333">0</span>')) : ''}
|
||
${apiOnly ? `<div style="font-size:10px;color:#333;margin-top:6px;">Watchdog state unavailable — SSH not configured</div>` : ''}
|
||
</div>`;
|
||
}
|
||
|
||
// ── Docker watchdog state card ────────────────────────────────────────────────
|
||
function _dockerCard(node, cfg) {
|
||
const st = node.states;
|
||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Docker Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||
|
||
const strikes = Object.entries(st.ctr_strikes || {});
|
||
const skiplist = st.skiplist || [];
|
||
const restarts = (st.restarts || []).slice(0, 10);
|
||
const daemonCls = st.daemon_strikes > 0 ? 'err' : 'ok';
|
||
const allOk = strikes.length === 0 && skiplist.length === 0 && st.daemon_strikes === 0;
|
||
|
||
// Group restarts by container for last-24h summary
|
||
const rCounts = {};
|
||
for (const r of restarts) {
|
||
rCounts[r.name] = (rCounts[r.name] || 0) + 1;
|
||
}
|
||
|
||
let strikesHtml = '';
|
||
if (strikes.length === 0 && st.daemon_strikes === 0) {
|
||
strikesHtml = '<div style="color:#333;font-size:11px;">0 active strikes</div>';
|
||
} else {
|
||
if (st.daemon_strikes > 0) {
|
||
strikesHtml += `<div class="vv-wd-strike-row"><span class="vv-wd-strike-name">daemon</span><span class="vv-wd-strike-cnt">${st.daemon_strikes}</span></div>`;
|
||
}
|
||
for (const [name, cnt] of strikes) {
|
||
strikesHtml += `<div class="vv-wd-strike-row">
|
||
<span class="vv-wd-strike-name">${vvEscHtml(name)}</span>
|
||
<span class="vv-wd-strike-cnt">${cnt} / ${cfg.cpu_fail_lim}</span>
|
||
${_why(name + ' strikes', 'The docker watchdog has ' + cnt + ' strike(s) against '
|
||
+ name + '. What is failing its check?')}
|
||
</div>`;
|
||
}
|
||
}
|
||
|
||
// Shown with the time left, not just the name. A mute whose remaining time is invisible is
|
||
// indistinguishable from the permanent exemptions it was built to replace — the countdown is
|
||
// the entire difference, so it is the part that has to be on screen.
|
||
const mutes = st.mutes || {};
|
||
const muteNames = Object.keys(mutes);
|
||
const muteHtml = muteNames.length === 0
|
||
? '<div style="color:#333;font-size:11px;">none</div>'
|
||
: `<div class="vv-wd-pill-row">${muteNames.map(n => {
|
||
const m = mutes[n];
|
||
const left = m.left >= 3600 ? Math.round(m.left / 3600) + 'h'
|
||
: m.left >= 60 ? Math.round(m.left / 60) + 'm'
|
||
: m.left + 's';
|
||
return `<span class="vv-wd-pill warn" title="${vvEscAttr(m.reason || 'no reason given')}">`
|
||
+ `${vvEscHtml(n)} · ${left}</span>`;
|
||
}).join('')}</div>`;
|
||
|
||
let skipHtml = '';
|
||
if (skiplist.length === 0) {
|
||
skipHtml = '<div style="color:#333;font-size:11px;">empty</div>';
|
||
} else {
|
||
skipHtml = `<div class="vv-wd-pill-row">${skiplist.map(c => _pill(c, 'err')).join('')}</div>`;
|
||
}
|
||
|
||
let restartHtml = '';
|
||
const rcEntries = Object.entries(rCounts);
|
||
if (rcEntries.length === 0) {
|
||
restartHtml = '<div style="color:#333;font-size:11px;">none (24h)</div>';
|
||
} else {
|
||
restartHtml = rcEntries.map(([n, c]) =>
|
||
`<div class="vv-wd-row">
|
||
<span class="vv-wd-lbl">${vvEscHtml(n)}</span>
|
||
<span class="vv-wd-val" style="color:${c >= cfg.restart_limit ? '#ef5350' : '#ffb74d'}">${c}×</span>
|
||
${_why(n + ' restarts', n + ' has been restarted ' + c
|
||
+ ' time(s) in the last 24 hours. Why does it keep needing a restart?')}
|
||
</div>`
|
||
).join('');
|
||
}
|
||
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-sec">Docker Watchdog</div>
|
||
<div style="display:flex;gap:6px;margin-bottom:8px;">
|
||
${_pill(st.daemon_restart ? 'daemon restarted' : 'daemon ok', daemonCls)}
|
||
${allOk ? _pill('all clear', 'ok') : ''}
|
||
</div>
|
||
<div class="vv-wd-sec">Strikes</div>
|
||
${strikesHtml}
|
||
<hr class="vv-wd-sep">
|
||
<div class="vv-wd-sec">Skip list</div>
|
||
${skipHtml}
|
||
<hr class="vv-wd-sep">
|
||
<div class="vv-wd-sec">Muted</div>
|
||
${muteHtml}
|
||
<hr class="vv-wd-sep">
|
||
<div class="vv-wd-sec">Restarts (24h)</div>
|
||
${restartHtml}
|
||
</div>`;
|
||
}
|
||
|
||
// ── Stability / reboot card ───────────────────────────────────────────────────
|
||
function _stabilityCard(node, cfg) {
|
||
const st = node.states;
|
||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Stability</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||
|
||
const reboots = st.reboots || [];
|
||
const sysStr = Object.entries(st.sys_strikes || {});
|
||
const rebootCls = reboots.length >= cfg.reboot_limit ? 'err' : reboots.length > 0 ? 'warn' : 'ok';
|
||
|
||
let sysHtml = '';
|
||
if (sysStr.length === 0) {
|
||
sysHtml = '<div style="color:#333;font-size:11px;">0 active strikes</div>';
|
||
} else {
|
||
sysHtml = sysStr.map(([k, v]) =>
|
||
`<div class="vv-wd-strike-row">
|
||
<span class="vv-wd-strike-name" style="font-size:10px;">${vvEscHtml(k.replace(/_/g,' '))}</span>
|
||
<span class="vv-wd-strike-cnt">${v}</span>
|
||
${_why(k.replace(/_/g,' ') + ' strikes', 'The stability watchdog has ' + v
|
||
+ ' strike(s) for ' + k.replace(/_/g,' ') + '. What is it detecting?')}
|
||
</div>`
|
||
).join('');
|
||
}
|
||
|
||
// The one entry on this page that is always worth explaining, so the button is on the heading
|
||
// rather than each row — an unattended reboot is a single question, not one per occurrence.
|
||
const rebootHtml = reboots.length === 0
|
||
? '<div style="color:#333;font-size:11px;">none (12h)</div>'
|
||
: reboots.map(ts => `<div class="vv-wd-reboot-ts">${_relTime(ts)}</div>`).join('')
|
||
+ `<div style="margin-top:5px;">${_why(node.id + ' reboots',
|
||
'The stability watchdog rebooted ' + node.id + ' ' + reboots.length
|
||
+ ' time(s) in the last ' + cfg.reboot_window + ' hours. What forced it?')}</div>`;
|
||
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-sec">Stability</div>
|
||
<div style="display:flex;gap:6px;margin-bottom:8px;">
|
||
${_pill(`${reboots.length} / ${cfg.reboot_limit} reboots`, rebootCls)}
|
||
${_pill(`${cfg.reboot_window}h window`, '')}
|
||
</div>
|
||
<div class="vv-wd-sec">Strikes</div>
|
||
${sysHtml}
|
||
<hr class="vv-wd-sep">
|
||
<div class="vv-wd-sec">Reboot history</div>
|
||
${rebootHtml}
|
||
</div>`;
|
||
}
|
||
|
||
// ── Config inventory card ─────────────────────────────────────────────────────
|
||
function _configCard(node) {
|
||
const cfg = node.config || {};
|
||
const mon = Object.entries(cfg.monitored || {});
|
||
const req = cfg.required || [];
|
||
const ign = cfg.ignore || [];
|
||
const paus = cfg.pause_list|| [];
|
||
const stop = cfg.stop_list || [];
|
||
const crit = cfg.critical || [];
|
||
|
||
// Only this host's lists can be checked — VvWdInstalled is what docker on THIS machine reports,
|
||
// and marking a partner's entry dead against it would be a guess dressed as a fact.
|
||
const P = node.is_me ? _pillK : _pill;
|
||
const K = name => node.is_me && !_known(name);
|
||
|
||
const monHtml = mon.length === 0
|
||
? '<div style="color:#333;font-size:11px;">none</div>'
|
||
: mon.map(([name, mb]) => {
|
||
const gb = (mb / 1024).toFixed(0);
|
||
return `<div class="vv-wd-ctr-row${K(name) ? ' missing' : ''}"`
|
||
+ `${K(name) ? ' title="No container by this name is installed — this limit does nothing"' : ''}>`
|
||
+ `<span class="vv-wd-ctr-name">${vvEscHtml(name)}</span>`
|
||
+ `<span class="vv-wd-ctr-lim">${K(name) ? 'not installed' : gb + ' GB'}</span></div>`;
|
||
}).join('');
|
||
|
||
const reqHtml = req.length === 0
|
||
? '<div style="color:#333;font-size:11px;">none</div>'
|
||
: `<div class="vv-wd-pill-row">${req.map(c => P(c, '')).join('')}</div>`;
|
||
|
||
const ignHtml = ign.length === 0
|
||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||
// Never badged, unlike the lists above. An ignore entry naming a container that is gone costs
|
||
// nothing — something that does not exist is not being watched either way — and seven dashed
|
||
// pills here would teach the eye to skip a badge that has to be believed on the stop list.
|
||
: `<div class="vv-wd-pill-row">${ign.map(c => _pill(c, '')).join('')}</div>`;
|
||
|
||
const pausHtml = paus.length === 0
|
||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||
: `<div class="vv-wd-pill-row">${paus.map(c => P(c, 'warn')).join('')}</div>`;
|
||
|
||
const stopHtml = stop.length === 0
|
||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||
: `<div class="vv-wd-pill-row">${stop.map(c => P(c, 'err')).join('')}</div>`;
|
||
|
||
return `<div class="vv-wd-card" style="grid-column:span 4;">
|
||
<div style="display:flex;gap:6px;align-items:center;margin-bottom:8px;">
|
||
<span class="vv-wd-node-id">${vvEscHtml(node.id)}</span>
|
||
<span style="font-size:11px;color:#3a3a3a;">${vvEscHtml(node.hostname)}</span>
|
||
</div>
|
||
<div style="display:grid;grid-template-columns:1fr 1fr;gap:10px;">
|
||
<div>
|
||
<div class="vv-wd-sec">Mem limits (Tier 1)</div>
|
||
${monHtml}
|
||
<div style="height:8px"></div>
|
||
<div class="vv-wd-sec">Required</div>
|
||
${reqHtml}
|
||
</div>
|
||
<div>
|
||
<div class="vv-wd-sec">Pause at medium pressure</div>
|
||
${pausHtml}
|
||
<div style="height:8px"></div>
|
||
<div class="vv-wd-sec">Stop at hard pressure</div>
|
||
${stopHtml}
|
||
<div style="height:8px"></div>
|
||
<div class="vv-wd-sec">Scan ignore</div>
|
||
${ignHtml}
|
||
</div>
|
||
</div>
|
||
</div>`;
|
||
}
|
||
|
||
// ── Storage watchdog card ─────────────────────────────────────────────────────
|
||
function _storageCard(node, cfg) {
|
||
const st = node.states;
|
||
const nodeCfg = node.config || {};
|
||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Storage Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||
|
||
const storWd = st.storage_wd || {};
|
||
const growthStr = Object.entries(storWd.growth_strikes || {});
|
||
const logStr = Object.entries(storWd.log_strikes || {});
|
||
const totalIssues = growthStr.length + logStr.length;
|
||
const allClear = totalIssues === 0;
|
||
|
||
// Baseline info
|
||
const bCount = storWd.baseline_count ?? 0;
|
||
const bAge = storWd.baseline_age_sec;
|
||
let baselineNote = bCount > 0
|
||
? `${bCount} containers tracked`
|
||
: 'no baseline yet (builds after first cycle)';
|
||
if (bAge != null && bCount > 0) {
|
||
const bAgeStr = bAge < 120 ? bAge + 's ago' : bAge < 3600 ? Math.floor(bAge/60) + 'm ago' : Math.floor(bAge/3600) + 'h ago';
|
||
baselineNote += ` · updated ${bAgeStr}`;
|
||
}
|
||
|
||
// Suppress ceilings configured for this host
|
||
const sizes = Object.entries(nodeCfg.appdata_sizes || {});
|
||
const sizesHtml = sizes.length
|
||
? sizes.map(([c, mb]) => _pill(c + ' <' + Math.round(mb/1024) + 'GB', '')).join('')
|
||
: '';
|
||
|
||
let growthHtml = '';
|
||
if (growthStr.length === 0) {
|
||
growthHtml = '<div style="color:#333;font-size:11px;">no active strikes</div>';
|
||
} else {
|
||
growthHtml = growthStr.map(([name, cnt]) =>
|
||
`<div class="vv-wd-strike-row">
|
||
<span class="vv-wd-strike-name">${vvEscHtml(name)}</span>
|
||
<span class="vv-wd-strike-cnt">${cnt} / ${cfg.stor_strike_lim}</span>
|
||
${_why(name + ' appdata growth', name + ' has ' + cnt
|
||
+ ' growth strike(s) — its appdata is growing more than '
|
||
+ cfg.growth_gb + 'GB per cycle. What is filling up?')}
|
||
</div>`
|
||
).join('');
|
||
}
|
||
|
||
let logHtml = '';
|
||
if (logStr.length === 0) {
|
||
logHtml = '<div style="color:#333;font-size:11px;">no active strikes</div>';
|
||
} else {
|
||
logHtml = logStr.map(([key, cnt]) => {
|
||
const display = key.length > 36 ? '…' + key.slice(-36) : key;
|
||
return `<div class="vv-wd-strike-row">
|
||
<span class="vv-wd-strike-name" style="font-size:10px;" title="${vvEscAttr(key)}">${vvEscHtml(display)}</span>
|
||
<span class="vv-wd-strike-cnt">${cnt} / ${cfg.stor_strike_lim}</span>
|
||
${_why('oversized log ' + key, key + ' has ' + cnt
|
||
+ ' log-size strike(s) against a ' + cfg.log_max_gb
|
||
+ 'GB ceiling. What is writing to it?')}
|
||
</div>`;
|
||
}).join('');
|
||
}
|
||
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-sec">Storage Watchdog</div>
|
||
<div style="display:flex;gap:5px;flex-wrap:wrap;margin-bottom:6px;">
|
||
${allClear ? _pill('all clear', 'ok') : _pill(totalIssues + ' active strike' + (totalIssues !== 1 ? 's' : ''), 'warn')}
|
||
${_pill('growth >' + cfg.growth_gb + 'GB/cycle', '')}
|
||
${_pill('log max ' + cfg.log_max_gb + 'GB', '')}
|
||
${cfg.truncate_logs ? _pill('auto-truncate on', 'ok') : _pill('auto-truncate off', '')}
|
||
</div>
|
||
<div style="font-size:10px;color:#3a3a3a;margin-bottom:8px;">${baselineNote}</div>
|
||
<div style="display:grid;grid-template-columns:1fr 1fr;gap:12px;">
|
||
<div>
|
||
<div class="vv-wd-sec">Growth strikes</div>
|
||
${growthHtml}
|
||
</div>
|
||
<div>
|
||
<div class="vv-wd-sec">Log size strikes</div>
|
||
${logHtml}
|
||
</div>
|
||
</div>
|
||
${sizes.length ? `<hr class="vv-wd-sep"><div class="vv-wd-sec">Suppress ceilings (this host)</div><div class="vv-wd-pill-row">${sizesHtml}</div>` : ''}
|
||
</div>`;
|
||
}
|
||
|
||
// ── Network watchdog card ─────────────────────────────────────────────────────
|
||
function _networkCard(node, cfg) {
|
||
const st = node.states;
|
||
const nodeCfg = node.config || {};
|
||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Network Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||
|
||
const netWd = st.network_wd || {};
|
||
const npmStr = netWd.npm_strikes ?? 0;
|
||
const npmCls = npmStr >= cfg.npm_strike_lim ? 'err' : npmStr > 0 ? 'warn' : 'ok';
|
||
const ddnsDomain = nodeCfg.ddns_domain || '';
|
||
const ddnsCtr = nodeCfg.ddns_container || '';
|
||
const npmUrl = nodeCfg.npm_url || '';
|
||
|
||
const ddnsHtml = ddnsDomain
|
||
? `${_row('DDNS domain', `<span style="color:#888;">${vvEscHtml(ddnsDomain)}</span>`)}
|
||
${ddnsCtr ? _row('DDNS container', `<span style="color:#888;">${vvEscHtml(ddnsCtr)}</span>`) : ''}`
|
||
: _row('DDNS', '<span style="color:#444;">not configured for this host</span>');
|
||
|
||
const npmHtml = npmUrl
|
||
? `${_row('NPM URL', `<span style="color:#888;font-size:10px;">${vvEscHtml(npmUrl)}</span>`)}
|
||
${_row('NPM strikes', `<span class="vv-wd-pill ${npmCls}" style="font-size:10px;">${npmStr} / ${cfg.npm_strike_lim}</span>`
|
||
+ (npmStr > 0 ? ' ' + _why('NPM strikes on ' + node.id,
|
||
'The network watchdog has ' + npmStr + ' strike(s) against NPM on '
|
||
+ node.id + '. What check is failing?') : ''))}`
|
||
: _row('NPM check', '<span style="color:#444;">not configured for this host</span>');
|
||
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-sec">Network Watchdog</div>
|
||
<div style="display:flex;gap:5px;flex-wrap:wrap;margin-bottom:8px;">
|
||
${cfg.net_wd_enabled ? _pill('enabled', 'ok') : _pill('disabled', '')}
|
||
${_pill('Tailscale check ' + (cfg.ts_check ? 'on' : 'off'), cfg.ts_check ? '' : '')}
|
||
${npmUrl ? _pill('NPM ' + npmStr + '/' + cfg.npm_strike_lim + ' strikes', npmCls) : ''}
|
||
</div>
|
||
${ddnsHtml}
|
||
<hr class="vv-wd-sep">
|
||
${npmHtml}
|
||
</div>`;
|
||
}
|
||
|
||
// ── Main render ───────────────────────────────────────────────────────────────
|
||
function _render(data) {
|
||
const nodes = data.nodes || [];
|
||
const cfg = data.cfg || {};
|
||
let html = '';
|
||
|
||
// Per-host: pressure alert (if active) then all 5 watchdog cards in one equal-spaced row
|
||
for (const node of nodes) {
|
||
html += _pressureCard(node);
|
||
html += `<div class="vv-wd-host-row">
|
||
${_systemCard(node, cfg)}
|
||
${_dockerCard(node, cfg)}
|
||
${_stabilityCard(node, cfg)}
|
||
${_storageCard(node, cfg)}
|
||
${_networkCard(node, cfg)}
|
||
</div>`;
|
||
}
|
||
|
||
// Config inventory — separate row, each node span 4 (2 nodes = full row)
|
||
for (const node of nodes) html += _configCard(node);
|
||
|
||
if (!html) html = '<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">No nodes configured.</div>';
|
||
|
||
document.getElementById('vv-wd-grid').innerHTML = html;
|
||
|
||
const ts = data.ts
|
||
? new Date(data.ts * 1000).toLocaleString([], {
|
||
month:'numeric', day:'numeric', year:'numeric',
|
||
hour:'2-digit', minute:'2-digit', second:'2-digit'})
|
||
: '';
|
||
document.getElementById('vv-wd-ts').textContent = ts ? 'Updated: ' + ts : '';
|
||
}
|
||
|
||
function vvWdLoad() {
|
||
fetch('/plugins/varaverk/api/watchdog.php')
|
||
.then(r => r.json())
|
||
.then(_render)
|
||
.catch(() => {
|
||
document.getElementById('vv-wd-grid').innerHTML =
|
||
'<div style="grid-column:1/-1;color:#555;font-size:12px;padding:24px 0;text-align:center;">Error loading watchdog data — check API</div>';
|
||
});
|
||
}
|
||
|
||
// One listener on the grid rather than a handler per button. The grid is replaced wholesale every
|
||
// 30 seconds, so anything bound to a button inside it would be thrown away on the next poll — and
|
||
// re-binding after each render is the version of this that quietly stops working the day a card
|
||
// gains a button and nobody updates the binder.
|
||
document.getElementById('vv-wd-grid').addEventListener('click', ev => {
|
||
const b = ev.target.closest('.vv-why-btn');
|
||
if (!b) return;
|
||
vvWdWhy(b.dataset.label || 'this watchdog', b.dataset.q || 'What does this mean?');
|
||
});
|
||
|
||
// Before the first load, not after. _why() renders nothing while vvWdChat is null, so a render
|
||
// that beat the instance into existence would draw the whole page without a single Why? button
|
||
// and not correct itself until the next poll thirty seconds later.
|
||
if (document.getElementById('vv-wd-ai-chat')) {
|
||
vvWdChat = VvAiChat({
|
||
prefix: 'vv-wd-ai',
|
||
profile: 'varaverk',
|
||
scopeLabel: 'Watchdog',
|
||
// Read at send time, not captured: a Why? moves the scope to the orchestrator's log, and the
|
||
// question is sent from that same click.
|
||
scope: () => vvWdScope,
|
||
// Same reason the Monitor card pins its own: without it the card resumes whatever thread was
|
||
// last touched anywhere, so opening this tab could land it mid-conversation from the Scheduler
|
||
// under a profile this page never offers.
|
||
resumeProfile: 'varaverk',
|
||
// Reasoning is worth watching for a diagnosis and noise for a definition — same rule the
|
||
// Scheduler applies.
|
||
think: p => p === 'troubleshoot',
|
||
empty: 'Ask about a strike, a restart, or a threshold on this page. '
|
||
+ 'Why? on any strike asks against the orchestrator\'s log.',
|
||
});
|
||
}
|
||
|
||
vvWdLoad();
|
||
setInterval(vvWdLoad, 30000);
|
||
|
||
// ── Editable container lists ──────────────────────────────────────────────────
|
||
// Structured rows over the same guarded write path the settings card uses: the value posted is the
|
||
// array *body*, and vv_conf_write_file() splices it between the parens it already found. Nothing
|
||
// here writes conf directly, so the lock, the backup, the `bash -n` check, the read-back and the
|
||
// rollback all still apply exactly as they do to a typed edit.
|
||
//
|
||
// Comments are carried per row rather than regenerated. Every one of these entries has a reason
|
||
// written beside it and rebuilding the array from names and numbers alone would erase all of them
|
||
// in one save — the sort of loss that is invisible until the day someone asks why Emby is 24GB.
|
||
(function () {
|
||
const HOST = document.getElementById('vv-wd-lists');
|
||
if (!HOST || !window.VvWdLists) return;
|
||
const API = '/plugins/varaverk/api/confform.php';
|
||
|
||
const INDENT = ' ';
|
||
// ["Name"]=1234 # why — value may be quoted or bare, and the reason may be absent
|
||
const MAP_RE = /^\s*\["([^"]+)"\]\s*=\s*"?([^"#]*?)"?\s*(?:#\s*(.*))?$/;
|
||
// "Name" # why
|
||
const LIST_RE = /^\s*"([^"]+)"\s*(?:#\s*(.*))?$/;
|
||
|
||
function parseBody(body, kind) {
|
||
const out = [];
|
||
for (const raw of String(body || '').split('\n')) {
|
||
if (!raw.trim() || raw.trim().startsWith('#')) continue;
|
||
const m = raw.match(kind === 'map' ? MAP_RE : LIST_RE);
|
||
if (!m) continue;
|
||
out.push(kind === 'map'
|
||
? { name: m[1], value: (m[2] || '').trim(), note: (m[3] || '').trim() }
|
||
: { name: m[1], value: '', note: (m[2] || '').trim() });
|
||
}
|
||
return out;
|
||
}
|
||
|
||
function buildBody(rows, kind) {
|
||
return rows.map(r => {
|
||
const head = kind === 'map' ? `["${r.name}"]=${r.value}` : `"${r.name}"`;
|
||
return INDENT + (r.note ? head + ' # ' + r.note : head);
|
||
}).join('\n');
|
||
}
|
||
|
||
VvWdLists.forEach((def, idx) => {
|
||
const card = document.createElement('div');
|
||
card.className = 'vv-wl-card';
|
||
HOST.appendChild(card);
|
||
|
||
let rows = parseBody(def.body, def.kind);
|
||
const original = buildBody(rows, def.kind);
|
||
|
||
function draw() {
|
||
card.innerHTML = '';
|
||
const h = document.createElement('div');
|
||
h.className = 'vv-wl-h';
|
||
h.innerHTML = `<span class="vv-wl-t">${vvEscHtml(def.title)}</span>`
|
||
+ `<span class="vv-wl-n">${rows.filter(r => !r.gone).length}</span>`;
|
||
card.appendChild(h);
|
||
|
||
const hint = document.createElement('div');
|
||
hint.className = 'vv-wl-hint';
|
||
hint.textContent = def.hint;
|
||
card.appendChild(hint);
|
||
|
||
rows.forEach((r, i) => {
|
||
const el = document.createElement('div');
|
||
el.className = 'vv-wl-row' + (r.gone ? ' gone' : '');
|
||
const gb = def.kind === 'map' && +r.value > 0
|
||
? `<span class="vv-wl-gb">${(+r.value / 1024).toFixed(1)} GB</span>` : '';
|
||
el.innerHTML =
|
||
`<input class="vv-wl-name" value="${vvEscAttr(r.name)}" spellcheck="false">`
|
||
+ `<span class="vv-wl-gone-t" style="display:none">not installed</span>`
|
||
+ (def.kind === 'map'
|
||
? `<input class="vv-wl-val" value="${vvEscAttr(r.value)}" inputmode="numeric">`
|
||
+ `<span class="vv-wl-unit">${vvEscHtml(def.unit)}</span>${gb}`
|
||
: '')
|
||
+ `<button class="vv-wl-x" title="${r.gone ? 'Keep this entry' : 'Remove this entry'}">`
|
||
+ (r.gone ? '↺' : '×') + '</button>';
|
||
|
||
const [nameEl, valEl] = [el.querySelector('.vv-wl-name'), el.querySelector('.vv-wl-val')];
|
||
const tagEl = el.querySelector('.vv-wl-gone-t');
|
||
// Re-checked on every keystroke, so correcting a stale name clears the badge as soon as it
|
||
// matches something real — which is the whole workflow this is meant to support. Blank
|
||
// rows are exempt: a half-typed new entry is not a dead one.
|
||
const tag = () => {
|
||
const n = nameEl.value.trim();
|
||
tagEl.style.display = (def.check && n !== '' && !_known(n)) ? '' : 'none';
|
||
};
|
||
tag();
|
||
nameEl.addEventListener('input', () => { r.name = nameEl.value.trim(); tag(); mark(); });
|
||
if (valEl) valEl.addEventListener('input', () => { r.value = valEl.value.trim(); mark(); });
|
||
// Marked rather than spliced out, so a mis-click is one click back and the reason written
|
||
// beside the entry is still there to read while deciding.
|
||
el.querySelector('.vv-wl-x').addEventListener('click', () => {
|
||
rows[i].gone = !rows[i].gone; draw();
|
||
});
|
||
card.appendChild(el);
|
||
if (r.note) {
|
||
const n = document.createElement('div');
|
||
n.className = 'vv-wl-note';
|
||
n.textContent = r.note;
|
||
card.appendChild(n);
|
||
}
|
||
});
|
||
|
||
const foot = document.createElement('div');
|
||
foot.className = 'vv-wl-foot';
|
||
foot.innerHTML = `<button class="vv-btn-sm" data-a="add">+ Add</button>`
|
||
+ `<span class="vv-wl-sum"></span>`
|
||
+ `<button class="vv-btn-sm" data-a="save" disabled>Save</button>`;
|
||
card.appendChild(foot);
|
||
foot.querySelector('[data-a="add"]').addEventListener('click', () => {
|
||
rows.push({ name: '', value: def.kind === 'map' ? '1024' : '', note: '' });
|
||
draw();
|
||
const inputs = card.querySelectorAll('.vv-wl-name');
|
||
if (inputs.length) inputs[inputs.length - 1].focus();
|
||
});
|
||
foot.querySelector('[data-a="save"]').addEventListener('click', save);
|
||
mark();
|
||
}
|
||
|
||
function live() { return rows.filter(r => !r.gone && r.name !== ''); }
|
||
|
||
function mark() {
|
||
const changed = buildBody(live(), def.kind) !== original;
|
||
const btn = card.querySelector('[data-a="save"]');
|
||
const sum = card.querySelector('.vv-wl-sum');
|
||
if (btn) btn.disabled = !changed;
|
||
if (sum && !sum.classList.contains('ok') && !sum.classList.contains('err'))
|
||
sum.textContent = changed ? 'unsaved' : '';
|
||
if (sum && changed) { sum.className = 'vv-wl-sum'; sum.textContent = 'unsaved'; }
|
||
}
|
||
|
||
function save() {
|
||
const keep = live();
|
||
const sum = card.querySelector('.vv-wl-sum');
|
||
const btn = card.querySelector('[data-a="save"]');
|
||
// Names are what get written between quotes in a bash array, so anything that could close
|
||
// one early or start a substitution is refused here rather than sent and rejected downstream.
|
||
const bad = keep.find(r => !/^[A-Za-z0-9._-]+$/.test(r.name))
|
||
|| (def.kind === 'map' ? keep.find(r => !/^\d+$/.test(r.value)) : null);
|
||
if (bad) {
|
||
sum.className = 'vv-wl-sum err';
|
||
sum.textContent = 'check "' + bad.name + '"';
|
||
return;
|
||
}
|
||
const dup = keep.map(r => r.name).find((n, i, a) => a.indexOf(n) !== i);
|
||
if (dup) { sum.className = 'vv-wl-sum err'; sum.textContent = dup + ' listed twice'; return; }
|
||
|
||
btn.disabled = true;
|
||
sum.className = 'vv-wl-sum';
|
||
sum.textContent = 'saving…';
|
||
// URLSearchParams, never FormData — a multipart POST to this plugin's endpoints hangs with
|
||
// no status ever returned.
|
||
fetch(API, { method: 'POST',
|
||
headers: { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' },
|
||
body: new URLSearchParams({ changes: JSON.stringify([{
|
||
file: def.file, key: def.key, type: def.type, value: buildBody(keep, def.kind),
|
||
}]) }) })
|
||
.then(r => r.text())
|
||
.then(t => {
|
||
let d;
|
||
try { d = JSON.parse(t); }
|
||
catch (_) {
|
||
throw new Error(t.trim() ? 'unparseable response'
|
||
: 'empty response — rejected before the endpoint ran');
|
||
}
|
||
if (!d.ok) throw new Error(d.error || 'save failed');
|
||
rows = keep.map(r => ({ ...r }));
|
||
def.body = buildBody(rows, def.kind);
|
||
draw();
|
||
const s = card.querySelector('.vv-wl-sum');
|
||
s.className = 'vv-wl-sum ok';
|
||
s.textContent = 'saved — takes effect next cycle';
|
||
})
|
||
.catch(e => {
|
||
const s = card.querySelector('.vv-wl-sum');
|
||
s.className = 'vv-wl-sum err';
|
||
s.textContent = String(e.message || e);
|
||
card.querySelector('[data-a="save"]').disabled = false;
|
||
});
|
||
}
|
||
|
||
draw();
|
||
});
|
||
})();
|
||
|
||
})();
|
||
</script>
|