Gmer4Lfe
2cd1384786
Every existing exemption lasts until someone remembers to undo it, and nobody does — Healarr has sat in a pressure list since it was uninstalled and seven ignore entries name containers that are gone. A mute states when it ends and then ends, capped by WATCHDOG_MUTE_MAX_HOURS so temporary is enforced rather than intended. Applied where IGNORE_MAP is built, so all five check sites inherit it, and shown with its countdown because an invisible suppression is the thing being fixed.
1079 lines
55 KiB
PHP
1079 lines
55 KiB
PHP
<?php
|
||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||
// PURPOSE
|
||
// Watchdog tab. Shows what every watchdog has recorded across all nodes — strike counts,
|
||
// action levels, paused and stopped containers, restart history, OOM and reboot counts.
|
||
//
|
||
// DESIGN PRINCIPLES
|
||
// State is a pure view. Everything the watchdogs have recorded arrives from one poll against
|
||
// api/watchdog.php, and all parsing lives in include/watchdog.php behind that endpoint, so the
|
||
// page cannot hold a second opinion about what a strike means.
|
||
//
|
||
// Configuration is not, and reads conf directly. The container lists and the settings card are
|
||
// rendered server-side from host*.conf at page load rather than from the polled payload: that
|
||
// payload is cached for up to 300s and drops the per-entry comments, which are where the reason
|
||
// for every number in these lists is written down.
|
||
//
|
||
// Shows every node, not just this one. A watchdog page that only covered the host you
|
||
// happened to open it on would miss exactly the node in trouble. Editing is local-host only
|
||
// though — the partner owns its own host*.conf.
|
||
//
|
||
// OPERATIONAL SAFEGUARDS
|
||
// Read-only about state. There is no control on this page that clears a strike, restarts a
|
||
// container, or resets a counter — those belong to the watchdog that owns the state. Changing
|
||
// which containers a watchdog acts on is a conf edit and goes through the guarded write path
|
||
// with everyone else's; it is not a way to reach into what a watchdog has already decided.
|
||
//
|
||
// That holds for the assistant too, and is why the Why? buttons open troubleshoot rather than
|
||
// repair. troubleshoot may read the machine and the orchestrator's log; it holds no conf_write
|
||
// and no way to act. A page whose stated contract is "changes nothing" must not carry the one
|
||
// profile that can — see include/ai_profiles.php for why that grant is deliberately narrow.
|
||
//
|
||
// A node that fails to report renders as unavailable rather than healthy. Absence of data
|
||
// is never drawn as an all-clear.
|
||
//
|
||
// RENDERS
|
||
// Per-node cards: resource / docker / system / storage / network watchdog state,
|
||
// strike pills, restart history, reboot and OOM counts
|
||
// Editable container lists for this host: Tier 1 memory limits, the two pressure lists, Required
|
||
// Assistant card, scoped to the Watchdog tab
|
||
// Watchdog conf sections, via the shared settings renderer
|
||
//
|
||
// DEPENDS ON
|
||
// api/watchdog.php polled every 30s → include/watchdog.php (served from a 300s cache)
|
||
// api/confform.php POST → the guarded conf write path, for the list editors
|
||
// include/ai_chat.php → the shared chat component
|
||
require_once dirname(__DIR__) . '/include/confui.php';
|
||
require_once dirname(__DIR__) . '/include/ai_chat.php';
|
||
// For vv_wd_installed_containers() only. The state on this page still arrives entirely through
|
||
// api/watchdog.php — this include buys one function, not a second collection path.
|
||
require_once dirname(__DIR__) . '/include/watchdog.php';
|
||
?>
|
||
<style>
|
||
.vv-wd-card { background:#161616;border:1px solid #2a2a2a;border-radius:6px;padding:10px;min-width:0; }
|
||
.vv-wd-sec { font-size:10px;font-weight:bold;color:#444;letter-spacing:.07em;text-transform:uppercase;margin-bottom:6px; }
|
||
.vv-wd-row { display:flex;justify-content:space-between;align-items:baseline;gap:6px;margin:2px 0; }
|
||
.vv-wd-lbl { font-size:11px;color:#444;white-space:nowrap; }
|
||
.vv-wd-val { font-size:12px;color:#bbb;text-align:right; }
|
||
.vv-wd-sep { border:none;border-top:1px solid #1e1e1e;margin:6px 0; }
|
||
.vv-wd-pill { font-size:10px;padding:1px 6px;border-radius:2px;background:#1e1e1e;color:#666;border:1px solid #272727; }
|
||
.vv-wd-pill.ok { background:#0d1f0d;color:#4caf50;border-color:#1a3a1a; }
|
||
.vv-wd-pill.warn { background:#1f1500;color:#ffb74d;border-color:#3a2800; }
|
||
.vv-wd-pill.err { background:#200d0d;color:#ef5350;border-color:#3a1a1a; }
|
||
.vv-wd-pill-row { display:flex;flex-wrap:wrap;gap:4px;margin-top:4px; }
|
||
.vv-wd-bar { height:5px;border-radius:2px;background:#1e1e1e;margin-top:3px;overflow:hidden; }
|
||
.vv-wd-bar-fill{ height:100%;border-radius:2px;transition:width .3s; }
|
||
.vv-wd-badge { font-size:11px;font-weight:bold;padding:2px 8px;border-radius:3px; }
|
||
.vv-wd-badge.ok { background:#0d1f0d;color:#4caf50; }
|
||
.vv-wd-badge.soft { background:#1f1f00;color:#cddc39; }
|
||
.vv-wd-badge.med { background:#1f1000;color:#ffb74d; }
|
||
.vv-wd-badge.hard { background:#200d0d;color:#ef5350; }
|
||
.vv-wd-node-h { display:flex;align-items:center;gap:8px;margin-bottom:10px; }
|
||
.vv-wd-node-id { font-size:12px;font-weight:bold;color:#666;letter-spacing:.06em;text-transform:uppercase; }
|
||
.vv-wd-dot { width:6px;height:6px;border-radius:50%;flex-shrink:0; }
|
||
.vv-wd-strike-row { display:flex;align-items:baseline;gap:6px;margin:2px 0; }
|
||
.vv-wd-strike-name{ font-size:11px;color:#777;flex:1; }
|
||
.vv-wd-strike-cnt { font-size:11px;font-weight:bold;color:#ffb74d; }
|
||
.vv-wd-reboot-ts { font-size:11px;color:#555;margin:1px 0; }
|
||
.vv-wd-ctr-row { display:flex;justify-content:space-between;align-items:baseline;margin:2px 0; }
|
||
.vv-wd-ctr-name{ font-size:11px;color:#888; }
|
||
.vv-wd-ctr-lim { font-size:11px;color:#555; }
|
||
.vv-wd-pressure{ grid-column:1/-1;border-color:#3a2000;background:#1a1000; }
|
||
|
||
/* ── Editable container lists ────────────────────────────────────────────────
|
||
Rows, not a textarea. The same values are reachable through the settings card below as raw
|
||
bash array bodies, which is fine for reading and unforgiving for changing one number. */
|
||
.vv-wl-card { background:#161616;border:1px solid #2a2a2a;border-radius:6px;padding:10px;min-width:0;
|
||
display:flex;flex-direction:column; }
|
||
.vv-wl-h { display:flex;align-items:baseline;gap:8px;margin-bottom:2px; }
|
||
.vv-wl-t { font-size:10px;font-weight:bold;color:#666;letter-spacing:.07em;text-transform:uppercase; }
|
||
.vv-wl-n { font-size:10px;color:#3a3a3a;margin-left:auto; }
|
||
.vv-wl-hint { font-size:10px;color:#3f3f3f;line-height:1.45;margin-bottom:8px; }
|
||
.vv-wl-row { display:flex;align-items:center;gap:5px;margin:2px 0; }
|
||
.vv-wl-row.gone .vv-wl-name { text-decoration:line-through;opacity:.4; }
|
||
.vv-wl-name { flex:1;min-width:0;background:#0d0d0d;border:1px solid #242424;color:#bbb;
|
||
font-size:11px;padding:3px 6px;border-radius:3px;font-family:inherit; }
|
||
.vv-wl-val { width:74px;flex-shrink:0;background:#0d0d0d;border:1px solid #242424;color:#ddd;
|
||
font-size:11px;padding:3px 6px;border-radius:3px;text-align:right; }
|
||
.vv-wl-unit { font-size:10px;color:#3a3a3a;flex-shrink:0;width:20px; }
|
||
.vv-wl-gb { font-size:10px;color:#4a4a4a;flex-shrink:0;width:48px;text-align:right; }
|
||
.vv-wl-x { background:none;border:none;color:#4a4a4a;cursor:pointer;font-size:13px;
|
||
line-height:1;padding:0 3px;flex-shrink:0; }
|
||
.vv-wl-x:hover{ color:#ef5350; }
|
||
.vv-wl-row.gone .vv-wl-x { color:#ef5350; }
|
||
.vv-wl-note { font-size:10px;color:#3a3a3a;margin:0 0 4px 6px;line-height:1.4; }
|
||
.vv-wl-foot { display:flex;align-items:center;gap:6px;margin-top:auto;padding-top:8px; }
|
||
.vv-wl-sum { font-size:10px;color:#3a3a3a;flex:1;min-width:0; }
|
||
.vv-wl-sum.ok { color:#4caf50; }
|
||
.vv-wl-sum.err{ color:#ef5350; }
|
||
.vv-wl-dirty { border-color:#3a4a6a !important; }
|
||
/* An entry naming a container that is not installed. Dashed rather than red: it is not an error,
|
||
it is a line that does nothing — and on the pressure lists that is the difference between what
|
||
the card implies can be shed and what actually can be. */
|
||
.vv-wl-gone-t { font-size:9px;color:#7a5020;background:#1a1000;border:1px dashed #3a2800;
|
||
border-radius:2px;padding:0 4px;flex-shrink:0;white-space:nowrap; }
|
||
.vv-wd-pill.missing { opacity:.5;border-style:dashed;color:#7a5020;border-color:#3a2800; }
|
||
.vv-wd-ctr-row.missing .vv-wd-ctr-name { opacity:.5; }
|
||
|
||
/* One host per row — inner grid sizes all cards equally */
|
||
.vv-wd-host-row {
|
||
grid-column: 1 / -1;
|
||
display: grid;
|
||
grid-template-columns: repeat(5, 1fr);
|
||
gap: 12px;
|
||
}
|
||
@media (max-width: 900px) {
|
||
.vv-wd-host-row { grid-template-columns: 1fr; }
|
||
}
|
||
</style>
|
||
|
||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;padding:0 2px;">
|
||
<span style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;">Watchdog</span>
|
||
<span style="font-size:11px;color:#3a3a3a;" id="vv-wd-ts"></span>
|
||
</div>
|
||
|
||
<div id="vv-wd-grid" style="display:grid;grid-template-columns:repeat(8,1fr);gap:12px;">
|
||
<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">Loading…</div>
|
||
</div>
|
||
|
||
|
||
<?php
|
||
// ── Editable container lists ─────────────────────────────────────────────────────────────────
|
||
// Read from conf at render time rather than from the watchdog payload, for two reasons. The
|
||
// payload is cached for up to 300s, so a list edited here would keep showing its old contents
|
||
// until the writer's next pass; and it drops the per-entry comments, which is where the reason for
|
||
// every one of these numbers is written down — "24GB, raised 2026-07-11, was creeping toward the
|
||
// ceiling" is the whole value of the row.
|
||
//
|
||
// Local host only. These live in host*.conf, and the partner owns its own: writing HOST2's limits
|
||
// from HOST1 produces a change on the wrong side of a conf sync that is currently switched off
|
||
// anyway. The remote node's lists stay visible in the inventory card below, read-only.
|
||
$_vv_wd_me = strtoupper(vv_detect_host());
|
||
$_vv_wd_file = strtolower($_vv_wd_me) . '.conf';
|
||
$_vv_wd_defs = [
|
||
// First, because it is the one people come to this page looking for: the answer to "stop
|
||
// telling me about this container". Everything below tunes what a watchdog does; this is how
|
||
// you take something out of its hands entirely.
|
||
// 'check' — whether a name that matches no installed container is worth marking. True where a
|
||
// dead entry silently costs something: a Tier 1 limit that guards nothing, or a stop-list slot
|
||
// that sheds no memory at hard pressure. False here, because ignoring something that no longer
|
||
// exists is exactly what this list is for.
|
||
['key' => $_vv_wd_me . '_WATCHDOG_SCAN_IGNORE', 'kind' => 'list', 'check' => false,
|
||
'title' => 'Scan ignore', 'unit' => '',
|
||
'hint' => 'Left alone completely — no strikes, no restarts, no unhealthy or OOM reports. '
|
||
. 'For containers you knowingly run broken or stopped. It silences real faults as '
|
||
. 'readily as noise, so it is the wrong tool for one that is merely loud.'],
|
||
['key' => $_vv_wd_me . '_WATCHDOG_CONTAINERS', 'kind' => 'map', 'check' => true,
|
||
'title' => 'Memory limits (Tier 1)', 'unit' => 'MB',
|
||
'hint' => 'Restarted when it holds more than this. Tier 1 is per container and acts before '
|
||
. 'any pressure tier does.'],
|
||
['key' => $_vv_wd_me . '_RW_PAUSE_CONTAINERS', 'kind' => 'list', 'check' => true,
|
||
'title' => 'Pause at medium pressure', 'unit' => '',
|
||
'hint' => 'Paused — not stopped — while RAM sits below the medium threshold, and resumed on '
|
||
. 'recovery. Safe for anything that can lose a few minutes.'],
|
||
['key' => $_vv_wd_me . '_RW_STOP_CONTAINERS', 'kind' => 'list', 'check' => true,
|
||
'title' => 'Stop at hard pressure', 'unit' => '',
|
||
'hint' => 'Stopped outright at the hard threshold, and not restarted automatically. The last '
|
||
. 'step before the memory shutdown.'],
|
||
['key' => $_vv_wd_me . '_WATCHDOG_REQUIRED_CONTAINERS', 'kind' => 'list', 'check' => true,
|
||
'title' => 'Required', 'unit' => '',
|
||
'hint' => 'Must be running. Missing is a strike whether or not the container is being '
|
||
. 'watched for anything else.'],
|
||
];
|
||
$_vv_wd_fields = [];
|
||
foreach (vv_conf_all_groups($_vv_wd_file) as $_g) {
|
||
foreach ($_g['fields'] as $_f) $_vv_wd_fields[$_f['key']] = $_f;
|
||
}
|
||
$_vv_wd_lists = [];
|
||
foreach ($_vv_wd_defs as $_d) {
|
||
$_f = $_vv_wd_fields[$_d['key']] ?? null;
|
||
if (!$_f) continue; // key absent from this host's conf — draw nothing
|
||
$_vv_wd_lists[] = $_d + [
|
||
'file' => $_f['file'] ?? $_vv_wd_file,
|
||
'type' => $_f['type'], // assoc_array | array — the writer splices on this
|
||
'body' => $_f['value'],
|
||
];
|
||
}
|
||
?>
|
||
<?php
|
||
// Which container names still refer to something. Emitted once and read by both surfaces that
|
||
// draw these lists — the editors below and the inventory card in the polled grid — so the two
|
||
// cannot disagree about whether an entry is dead.
|
||
//
|
||
// Empty means "could not ask docker", not "nothing is installed", and every consumer checks the
|
||
// length before badging anything. A docker outage must not repaint the page as though every list
|
||
// were full of ghosts.
|
||
$_vv_wd_installed = vv_wd_installed_containers();
|
||
?>
|
||
<script>window.VvWdInstalled = <?= json_encode($_vv_wd_installed, JSON_UNESCAPED_SLASHES) ?>;</script>
|
||
<?php if ($_vv_wd_lists): ?>
|
||
<div id="vv-wd-lists" style="display:grid;grid-template-columns:repeat(auto-fit,minmax(320px,1fr));gap:12px;margin-top:12px;"></div>
|
||
<script>window.VvWdLists = <?= json_encode($_vv_wd_lists, JSON_UNESCAPED_SLASHES) ?>;</script>
|
||
<?php endif; ?>
|
||
|
||
<?php if (vv_ai_ui_on()): ?>
|
||
<div class="vv-card" id="vv-wd-ai-card" style="margin-top:12px;">
|
||
<?php
|
||
// All three, in this order, before the markup. vv_ai_chat_markup() emits only the boxes — the
|
||
// factory that brings them to life, the profile registry the picker draws from, and the store
|
||
// that keeps a thread across a reload are separate and none of them is implied by the others.
|
||
// Omitting them renders a chat that looks complete and dies on the first click with VvAiChat
|
||
// undefined, which is how the Scheduler's panel behaved before it was given the same three.
|
||
// Each is guarded internally, so calling them here costs nothing on a page that already has.
|
||
vv_ai_profiles_script();
|
||
vv_ai_chat_store_script();
|
||
vv_ai_chat_assets();
|
||
// Reads the page rather than sitting beside it. The scope is sent with every question, and the
|
||
// worker turns it into "they have Watchdog open — read bare references against it", so "why did
|
||
// it restart that" resolves here instead of asking which of forty containers is meant.
|
||
//
|
||
// Starts on the Varaverk assistant, not General Chat as the Monitor card does. The two pages ask
|
||
// different things: the dashboard is where an idle question gets typed, this is a page you only
|
||
// open because something struck. Answers about strikes and thresholds should come from this
|
||
// installation's own docs and state, with sources, rather than from the model's general
|
||
// impression of what a watchdog usually does.
|
||
//
|
||
// No 'height' triple — this card is in normal page flow, not a fixed panel, so the component's
|
||
// own defaults apply and the expand control still works against them. The Scheduler is the only
|
||
// placement whose size is a share of something and therefore has to be told.
|
||
vv_ai_chat_markup('vv-wd-ai', [
|
||
'profile' => 'varaverk',
|
||
'compact' => true,
|
||
'title' => 'Assistant',
|
||
'scopeLabel' => 'Watchdog',
|
||
'empty' => 'Ask about a strike, a restart, or a threshold on this page. '
|
||
. 'Why? on any strike asks against the orchestrator\'s log.',
|
||
'placeholder' => 'Ask about what is on this page…',
|
||
]); ?>
|
||
</div>
|
||
<?php endif; ?>
|
||
|
||
<?php
|
||
// The conf sections this page is about, drawn by the shared renderer. They were reachable
|
||
// only from the Settings tab's catch-all, which is a long way to go for a setting named
|
||
// after the page you are already looking at.
|
||
vv_conf_ui_card('vv-cf-watchdog', 'watchdog', 'Watchdog settings');
|
||
?>
|
||
<script>
|
||
(function() {
|
||
|
||
const GB = 1073741824;
|
||
|
||
// The assistant instance, once the card is on the page; null whenever AI is off. Every Why?
|
||
// button checks it, and none of them render without it — a control that opens a panel that does
|
||
// not exist is worse than no control.
|
||
let vvWdChat = null;
|
||
// What the chat says the operator is looking at. Starts as the tab itself and moves to the
|
||
// orchestrator's log when a Why? is pressed, because that is the only place any of this is
|
||
// written down: the watchdogs log through watchdog_orchestrator.sh and have no log files of
|
||
// their own — checked against /var/log/varaverk, where Watchdogs/ does not exist.
|
||
let vvWdScope = 'Watchdog';
|
||
const WD_LOG = 'Orchestrators/watchdog_orchestrator';
|
||
|
||
// Rendered as data attributes and read back by one delegated listener rather than written into an
|
||
// onclick. Container names and log paths reach these buttons from conf and from the filesystem, and
|
||
// a name containing a quote closes the attribute early and takes the rest of the handler with it.
|
||
// vvEscAttr is the one that escapes quotes; vvEscHtml deliberately does not.
|
||
function _why(label, q) {
|
||
if (!vvWdChat) return '';
|
||
return `<button class="vv-btn-sm vv-why-btn" title="Ask the assistant about this"
|
||
data-label="${vvEscAttr(label)}" data-q="${vvEscAttr(q)}">Why?</button>`;
|
||
}
|
||
|
||
function vvWdWhy(label, q) {
|
||
if (!vvWdChat || vvWdChat.busy()) return;
|
||
// Troubleshoot, never repair. This page changes nothing, and repair is the one profile that can
|
||
// — see the safeguard note in the header.
|
||
vvWdScope = WD_LOG;
|
||
vvWdChat.retarget('troubleshoot', label, 'now looking at ' + label);
|
||
const input = document.getElementById('vv-wd-ai-input');
|
||
if (input) input.value = q;
|
||
vvWdChat.send();
|
||
const card = document.getElementById('vv-wd-ai-card');
|
||
if (card) card.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
|
||
}
|
||
|
||
function _fmtBytes(b) {
|
||
if (!b) return '0';
|
||
if (b >= GB) return (b / GB).toFixed(1) + 'G';
|
||
if (b >= 1048576) return (b / 1048576).toFixed(0) + 'M';
|
||
return (b / 1024).toFixed(0) + 'K';
|
||
}
|
||
|
||
function _relTime(ts) {
|
||
if (!ts) return '—';
|
||
const s = Math.floor(Date.now() / 1000) - ts;
|
||
if (s < 60) return 'just now';
|
||
if (s < 3600) return Math.floor(s / 60) + 'm ago';
|
||
if (s < 86400) return Math.floor(s / 3600) + 'h ago';
|
||
return Math.floor(s / 86400) + 'd ago';
|
||
}
|
||
|
||
function _dur(s) {
|
||
const d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60);
|
||
if (d) return d + 'd ' + h + 'h';
|
||
if (h) return h + 'h ' + m + 'm';
|
||
return m + 'm';
|
||
}
|
||
|
||
function _row(lbl, val) {
|
||
return `<div class="vv-wd-row"><span class="vv-wd-lbl">${lbl}</span><span class="vv-wd-val">${val}</span></div>`;
|
||
}
|
||
|
||
function _bar(pct, col) {
|
||
return `<div class="vv-wd-bar"><div class="vv-wd-bar-fill" style="width:${Math.min(pct,100)}%;background:${col}"></div></div>`;
|
||
}
|
||
|
||
// Escapes its own label, so every caller passes plain text and none of them has to remember. The
|
||
// labels here are container names, log paths and DDNS domains — all of them values this page reads
|
||
// rather than writes, which is exactly the category that must not be trusted into markup.
|
||
function _pill(label, cls) {
|
||
return `<span class="vv-wd-pill ${cls}">${vvEscHtml(label)}</span>`;
|
||
}
|
||
|
||
// ── Does this name still refer to a container? ───────────────────────────────
|
||
// One resolver for both surfaces. Answers false only when it is certain: an empty install set
|
||
// means docker could not be asked, and every caller then badges nothing rather than everything.
|
||
const _installed = new Set(window.VvWdInstalled || []);
|
||
function _known(name) { return !_installed.size || _installed.has(String(name || '').trim()); }
|
||
|
||
// Applied to the lists whose entries are container names, and deliberately NOT to the appdata
|
||
// ceilings. Those are documented as containers but hold `7dtd`, which is an appdata folder — the
|
||
// container is 7DaysToDie. Until that rule is pinned down, badging them would mark a working
|
||
// entry dead, and a wrong badge on this page is worse than a missing one: the whole value of it
|
||
// is that it is believed.
|
||
function _pillK(label, cls) {
|
||
return _known(label) ? _pill(label, cls)
|
||
: `<span class="vv-wd-pill ${cls} missing" title="No container by this name is installed — this entry does nothing">${vvEscHtml(label)}</span>`;
|
||
}
|
||
|
||
function _levelLabel(level) {
|
||
return ['OK', 'SOFT', 'MEDIUM', 'HARD'][level] || '?';
|
||
}
|
||
function _levelCls(level) {
|
||
return ['ok', 'soft', 'med', 'hard'][level] || 'ok';
|
||
}
|
||
|
||
// ── Pressure alert card ───────────────────────────────────────────────────────
|
||
function _pressureCard(node) {
|
||
const st = node.states;
|
||
if (!st || st.rw_level === 0) return '';
|
||
const level = st.rw_level;
|
||
const cls = _levelCls(level);
|
||
const label = _levelLabel(level);
|
||
|
||
const paused = (st.rw_paused || []).filter(Boolean);
|
||
const stopped = (st.rw_stopped || []).filter(Boolean);
|
||
const pausedHtml = paused.length ? paused.map(c => _pill(c, 'warn')).join('') : '';
|
||
const stoppedHtml = stopped.length ? stopped.map(c => _pill(c, 'err')).join('') : '';
|
||
|
||
return `<div class="vv-wd-card vv-wd-pressure" style="grid-column:1/-1">
|
||
<div style="display:flex;align-items:center;gap:10px;margin-bottom:8px;">
|
||
<span class="vv-wd-badge ${cls}">PRESSURE ${label}</span>
|
||
<span style="font-size:11px;color:#7a5020;">${vvEscHtml(node.id)} (${vvEscHtml(node.hostname)})</span>
|
||
${_why(node.id + ' memory pressure',
|
||
'The resource watchdog is at ' + label + ' pressure on ' + node.id
|
||
+ '. What drove it there, and what did it pause or stop?')}
|
||
${st.mem_shutdown ? `<span class="vv-wd-badge hard" style="margin-left:auto;">MEM SHUTDOWN ACTIVE</span>` : ''}
|
||
</div>
|
||
${paused.length ? `<div style="margin-bottom:4px;"><span class="vv-wd-lbl">Paused:</span> <span class="vv-wd-pill-row" style="display:inline-flex;">${pausedHtml}</span></div>` : ''}
|
||
${stopped.length ? `<div><span class="vv-wd-lbl">Stopped:</span> <span class="vv-wd-pill-row" style="display:inline-flex;">${stoppedHtml}</span></div>` : ''}
|
||
</div>`;
|
||
}
|
||
|
||
// ── System health card ────────────────────────────────────────────────────────
|
||
function _systemCard(node, cfg) {
|
||
const sys = node.system;
|
||
if (!sys) {
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-node-h">
|
||
<span class="vv-wd-dot" style="background:#444"></span>
|
||
<span class="vv-wd-node-id">${vvEscHtml(node.id)}</span>
|
||
<span style="font-size:11px;color:#3a3a3a;">${vvEscHtml(node.hostname)}</span>
|
||
<span class="vv-wd-badge" style="margin-left:auto;background:#1a1a1a;color:#444">UNREACHABLE</span>
|
||
</div>
|
||
</div>`;
|
||
}
|
||
|
||
const memGb = sys.mem_avail / GB;
|
||
const memTotGb = sys.mem_total / GB;
|
||
const usedPct = memTotGb > 0 ? ((memTotGb - memGb) / memTotGb) * 100 : 0;
|
||
const memCol = memGb < cfg.sys_mem_gb ? '#ef5350'
|
||
: memGb < cfg.rw_hard_gb ? '#ef5350'
|
||
: memGb < cfg.rw_medium_gb ? '#ffb74d'
|
||
: memGb < cfg.rw_soft_gb ? '#cddc39' : '#4caf50';
|
||
|
||
const loadPct = sys.cores > 0 ? (sys.load1 / (sys.cores * cfg.rw_load_med)) * 100 : 0;
|
||
const loadCol = sys.load1 > sys.cores * cfg.rw_load_med ? '#ef5350'
|
||
: sys.load1 > sys.cores * cfg.rw_load_soft ? '#ffb74d'
|
||
: '#4caf50';
|
||
|
||
// The fourth member of SYSTEM_WATCHDOG_SCRIPTS, and the only one with nothing to show: it keeps
|
||
// the persistent backup of the partner's conf current, and writes no state until it does. Drawn
|
||
// as its gate rather than left off the card, because a chain member that is simply missing from
|
||
// the page reads as one that is fine — the same reason an unreachable node here says so instead
|
||
// of rendering healthy. Names the switch, so "why is it off" does not need a hunt through conf.
|
||
const confCacheHtml = !cfg.fallback_on
|
||
? '<span style="color:#555;">off — FALLBACK_ENABLED</span>'
|
||
: !cfg.conf_sync_on
|
||
? '<span style="color:#7a6a20;">off — CONF_SYNC_ENABLED</span>'
|
||
: '<span style="color:#4caf50;">active</span>';
|
||
|
||
// ── Is anything still running? ──────────────────────────────────────────────
|
||
// Every other number on this card is only meaningful if a cycle actually happened. If the
|
||
// orchestrator stopped, the strike maps stop changing and the whole page renders zeros, "all
|
||
// clear" pills and green dots — its healthiest possible appearance, describing a machine with
|
||
// no watchdogs at all. This is the one reading that distinguishes quiet from absent, and the
|
||
// page's own contract says absence must never be drawn as an all-clear.
|
||
//
|
||
// stability_watchdog.sh writes the heartbeat last in the chain, so a fresh value means a whole
|
||
// cycle finished rather than one that started and died halfway. Overdue is measured against the
|
||
// schedule in force (cfg.cycle_min, read from the cron), not a constant: one missed cycle is
|
||
// worth noticing, two is worth alarming about.
|
||
const cycleMin = cfg.cycle_min || 15;
|
||
const lastCyc = (node.states || {}).last_cycle || 0;
|
||
const cycAge = lastCyc ? Math.max(0, Math.floor(Date.now() / 1000) - lastCyc) : null;
|
||
const cycCol = cycAge == null ? '#555'
|
||
: cycAge > cycleMin * 60 * 3 ? '#ef5350'
|
||
: cycAge > cycleMin * 60 * 2 ? '#ffb74d' : '#4caf50';
|
||
const cycTxt = cycAge == null ? 'never' : _relTime(lastCyc);
|
||
const cycNote = cycAge == null ? ' — no cycle has completed'
|
||
: cycAge > cycleMin * 60 * 2 ? ' — overdue, expected every ' + cycleMin + 'm' : '';
|
||
|
||
const apiOnly = sys.api_only === true;
|
||
const daemonDot = sys.daemon_ok === null ? '#555' : sys.daemon_ok ? '#4caf50' : '#ef5350';
|
||
const daemonTxt = sys.daemon_ok === null ? '—' : sys.daemon_ok ? 'daemon ok' : 'daemon err';
|
||
|
||
const st = node.states || {};
|
||
const level = st.rw_level || 0;
|
||
const dotCol = level >= 3 ? '#ef5350' : level >= 2 ? '#ffb74d' : level >= 1 ? '#cddc39'
|
||
: apiOnly ? '#4a7a9b' // blue-grey: API-only, no watchdog state
|
||
: '#4caf50';
|
||
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-node-h">
|
||
<span class="vv-wd-dot" style="background:${dotCol}"></span>
|
||
<span class="vv-wd-node-id">${vvEscHtml(node.id)}</span>
|
||
<span style="font-size:11px;color:#3a3a3a;">${vvEscHtml(node.hostname)}</span>
|
||
<span class="vv-wd-badge ${apiOnly ? '' : _levelCls(level)}"
|
||
style="margin-left:auto;${apiOnly ? 'background:#0d1f2a;color:#4a9eff;' : ''}"
|
||
>${apiOnly ? 'API ONLY' : _levelLabel(level)}</span>
|
||
</div>
|
||
<div class="vv-wd-sec">System</div>
|
||
${!apiOnly ? _row('Last cycle', `<span style="color:${cycCol}">${vvEscHtml(cycTxt)}</span>`
|
||
+ `<span style="color:#3a3a3a;font-size:10px;">${vvEscHtml(cycNote)}</span>`) : ''}
|
||
${_row('RAM free', `<span style="color:${memCol}">${_fmtBytes(sys.mem_avail)}</span> / ${_fmtBytes(sys.mem_total)}`)}
|
||
${_bar(usedPct, memCol)}
|
||
<div style="display:flex;justify-content:space-between;margin-top:1px;font-size:10px;color:#333">
|
||
<span>free</span>
|
||
<span>${cfg.rw_soft_gb}G soft · ${cfg.rw_hard_gb}G hard · ${cfg.sys_mem_gb}G reboot</span>
|
||
</div>
|
||
<div style="height:5px"></div>
|
||
${!apiOnly ? `${_row('Load avg', `<span style="color:${loadCol}">${sys.load1.toFixed(2)}</span> / ${sys.cores} cores`)}
|
||
${_bar(loadPct, loadCol)}
|
||
<div style="height:5px"></div>` : ''}
|
||
${_row('Uptime', sys.uptime ? _dur(sys.uptime) : '—')}
|
||
${_row('Docker', `<span style="color:${daemonDot}">${daemonTxt}</span>`)}
|
||
${_row('Conf cache', confCacheHtml)}
|
||
${!apiOnly ? (sys.oom_count > 0 ? _row('OOM kills', `<span style="color:#ef5350">${sys.oom_count}</span>`) : _row('OOM kills', '<span style="color:#333">0</span>')) : ''}
|
||
${apiOnly ? `<div style="font-size:10px;color:#333;margin-top:6px;">Watchdog state unavailable — SSH not configured</div>` : ''}
|
||
</div>`;
|
||
}
|
||
|
||
// ── Docker watchdog state card ────────────────────────────────────────────────
|
||
function _dockerCard(node, cfg) {
|
||
const st = node.states;
|
||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Docker Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||
|
||
const strikes = Object.entries(st.ctr_strikes || {});
|
||
const skiplist = st.skiplist || [];
|
||
const restarts = (st.restarts || []).slice(0, 10);
|
||
const daemonCls = st.daemon_strikes > 0 ? 'err' : 'ok';
|
||
const allOk = strikes.length === 0 && skiplist.length === 0 && st.daemon_strikes === 0;
|
||
|
||
// Group restarts by container for last-24h summary
|
||
const rCounts = {};
|
||
for (const r of restarts) {
|
||
rCounts[r.name] = (rCounts[r.name] || 0) + 1;
|
||
}
|
||
|
||
let strikesHtml = '';
|
||
if (strikes.length === 0 && st.daemon_strikes === 0) {
|
||
strikesHtml = '<div style="color:#333;font-size:11px;">0 active strikes</div>';
|
||
} else {
|
||
if (st.daemon_strikes > 0) {
|
||
strikesHtml += `<div class="vv-wd-strike-row"><span class="vv-wd-strike-name">daemon</span><span class="vv-wd-strike-cnt">${st.daemon_strikes}</span></div>`;
|
||
}
|
||
for (const [name, cnt] of strikes) {
|
||
strikesHtml += `<div class="vv-wd-strike-row">
|
||
<span class="vv-wd-strike-name">${vvEscHtml(name)}</span>
|
||
<span class="vv-wd-strike-cnt">${cnt} / ${cfg.cpu_fail_lim}</span>
|
||
${_why(name + ' strikes', 'The docker watchdog has ' + cnt + ' strike(s) against '
|
||
+ name + '. What is failing its check?')}
|
||
</div>`;
|
||
}
|
||
}
|
||
|
||
// Shown with the time left, not just the name. A mute whose remaining time is invisible is
|
||
// indistinguishable from the permanent exemptions it was built to replace — the countdown is
|
||
// the entire difference, so it is the part that has to be on screen.
|
||
const mutes = st.mutes || {};
|
||
const muteNames = Object.keys(mutes);
|
||
const muteHtml = muteNames.length === 0
|
||
? '<div style="color:#333;font-size:11px;">none</div>'
|
||
: `<div class="vv-wd-pill-row">${muteNames.map(n => {
|
||
const m = mutes[n];
|
||
const left = m.left >= 3600 ? Math.round(m.left / 3600) + 'h'
|
||
: m.left >= 60 ? Math.round(m.left / 60) + 'm'
|
||
: m.left + 's';
|
||
return `<span class="vv-wd-pill warn" title="${vvEscAttr(m.reason || 'no reason given')}">`
|
||
+ `${vvEscHtml(n)} · ${left}</span>`;
|
||
}).join('')}</div>`;
|
||
|
||
let skipHtml = '';
|
||
if (skiplist.length === 0) {
|
||
skipHtml = '<div style="color:#333;font-size:11px;">empty</div>';
|
||
} else {
|
||
skipHtml = `<div class="vv-wd-pill-row">${skiplist.map(c => _pill(c, 'err')).join('')}</div>`;
|
||
}
|
||
|
||
let restartHtml = '';
|
||
const rcEntries = Object.entries(rCounts);
|
||
if (rcEntries.length === 0) {
|
||
restartHtml = '<div style="color:#333;font-size:11px;">none (24h)</div>';
|
||
} else {
|
||
restartHtml = rcEntries.map(([n, c]) =>
|
||
`<div class="vv-wd-row">
|
||
<span class="vv-wd-lbl">${vvEscHtml(n)}</span>
|
||
<span class="vv-wd-val" style="color:${c >= cfg.restart_limit ? '#ef5350' : '#ffb74d'}">${c}×</span>
|
||
${_why(n + ' restarts', n + ' has been restarted ' + c
|
||
+ ' time(s) in the last 24 hours. Why does it keep needing a restart?')}
|
||
</div>`
|
||
).join('');
|
||
}
|
||
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-sec">Docker Watchdog</div>
|
||
<div style="display:flex;gap:6px;margin-bottom:8px;">
|
||
${_pill(st.daemon_restart ? 'daemon restarted' : 'daemon ok', daemonCls)}
|
||
${allOk ? _pill('all clear', 'ok') : ''}
|
||
</div>
|
||
<div class="vv-wd-sec">Strikes</div>
|
||
${strikesHtml}
|
||
<hr class="vv-wd-sep">
|
||
<div class="vv-wd-sec">Skip list</div>
|
||
${skipHtml}
|
||
<hr class="vv-wd-sep">
|
||
<div class="vv-wd-sec">Muted</div>
|
||
${muteHtml}
|
||
<hr class="vv-wd-sep">
|
||
<div class="vv-wd-sec">Restarts (24h)</div>
|
||
${restartHtml}
|
||
</div>`;
|
||
}
|
||
|
||
// ── Stability / reboot card ───────────────────────────────────────────────────
|
||
function _stabilityCard(node, cfg) {
|
||
const st = node.states;
|
||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Stability</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||
|
||
const reboots = st.reboots || [];
|
||
const sysStr = Object.entries(st.sys_strikes || {});
|
||
const rebootCls = reboots.length >= cfg.reboot_limit ? 'err' : reboots.length > 0 ? 'warn' : 'ok';
|
||
|
||
let sysHtml = '';
|
||
if (sysStr.length === 0) {
|
||
sysHtml = '<div style="color:#333;font-size:11px;">0 active strikes</div>';
|
||
} else {
|
||
sysHtml = sysStr.map(([k, v]) =>
|
||
`<div class="vv-wd-strike-row">
|
||
<span class="vv-wd-strike-name" style="font-size:10px;">${vvEscHtml(k.replace(/_/g,' '))}</span>
|
||
<span class="vv-wd-strike-cnt">${v}</span>
|
||
${_why(k.replace(/_/g,' ') + ' strikes', 'The stability watchdog has ' + v
|
||
+ ' strike(s) for ' + k.replace(/_/g,' ') + '. What is it detecting?')}
|
||
</div>`
|
||
).join('');
|
||
}
|
||
|
||
// The one entry on this page that is always worth explaining, so the button is on the heading
|
||
// rather than each row — an unattended reboot is a single question, not one per occurrence.
|
||
const rebootHtml = reboots.length === 0
|
||
? '<div style="color:#333;font-size:11px;">none (12h)</div>'
|
||
: reboots.map(ts => `<div class="vv-wd-reboot-ts">${_relTime(ts)}</div>`).join('')
|
||
+ `<div style="margin-top:5px;">${_why(node.id + ' reboots',
|
||
'The stability watchdog rebooted ' + node.id + ' ' + reboots.length
|
||
+ ' time(s) in the last ' + cfg.reboot_window + ' hours. What forced it?')}</div>`;
|
||
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-sec">Stability</div>
|
||
<div style="display:flex;gap:6px;margin-bottom:8px;">
|
||
${_pill(`${reboots.length} / ${cfg.reboot_limit} reboots`, rebootCls)}
|
||
${_pill(`${cfg.reboot_window}h window`, '')}
|
||
</div>
|
||
<div class="vv-wd-sec">Strikes</div>
|
||
${sysHtml}
|
||
<hr class="vv-wd-sep">
|
||
<div class="vv-wd-sec">Reboot history</div>
|
||
${rebootHtml}
|
||
</div>`;
|
||
}
|
||
|
||
// ── Config inventory card ─────────────────────────────────────────────────────
|
||
function _configCard(node) {
|
||
const cfg = node.config || {};
|
||
const mon = Object.entries(cfg.monitored || {});
|
||
const req = cfg.required || [];
|
||
const ign = cfg.ignore || [];
|
||
const paus = cfg.pause_list|| [];
|
||
const stop = cfg.stop_list || [];
|
||
const crit = cfg.critical || [];
|
||
|
||
// Only this host's lists can be checked — VvWdInstalled is what docker on THIS machine reports,
|
||
// and marking a partner's entry dead against it would be a guess dressed as a fact.
|
||
const P = node.is_me ? _pillK : _pill;
|
||
const K = name => node.is_me && !_known(name);
|
||
|
||
const monHtml = mon.length === 0
|
||
? '<div style="color:#333;font-size:11px;">none</div>'
|
||
: mon.map(([name, mb]) => {
|
||
const gb = (mb / 1024).toFixed(0);
|
||
return `<div class="vv-wd-ctr-row${K(name) ? ' missing' : ''}"`
|
||
+ `${K(name) ? ' title="No container by this name is installed — this limit does nothing"' : ''}>`
|
||
+ `<span class="vv-wd-ctr-name">${vvEscHtml(name)}</span>`
|
||
+ `<span class="vv-wd-ctr-lim">${K(name) ? 'not installed' : gb + ' GB'}</span></div>`;
|
||
}).join('');
|
||
|
||
const reqHtml = req.length === 0
|
||
? '<div style="color:#333;font-size:11px;">none</div>'
|
||
: `<div class="vv-wd-pill-row">${req.map(c => P(c, '')).join('')}</div>`;
|
||
|
||
const ignHtml = ign.length === 0
|
||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||
// Never badged, unlike the lists above. An ignore entry naming a container that is gone costs
|
||
// nothing — something that does not exist is not being watched either way — and seven dashed
|
||
// pills here would teach the eye to skip a badge that has to be believed on the stop list.
|
||
: `<div class="vv-wd-pill-row">${ign.map(c => _pill(c, '')).join('')}</div>`;
|
||
|
||
const pausHtml = paus.length === 0
|
||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||
: `<div class="vv-wd-pill-row">${paus.map(c => P(c, 'warn')).join('')}</div>`;
|
||
|
||
const stopHtml = stop.length === 0
|
||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||
: `<div class="vv-wd-pill-row">${stop.map(c => P(c, 'err')).join('')}</div>`;
|
||
|
||
return `<div class="vv-wd-card" style="grid-column:span 4;">
|
||
<div style="display:flex;gap:6px;align-items:center;margin-bottom:8px;">
|
||
<span class="vv-wd-node-id">${vvEscHtml(node.id)}</span>
|
||
<span style="font-size:11px;color:#3a3a3a;">${vvEscHtml(node.hostname)}</span>
|
||
</div>
|
||
<div style="display:grid;grid-template-columns:1fr 1fr;gap:10px;">
|
||
<div>
|
||
<div class="vv-wd-sec">Mem limits (Tier 1)</div>
|
||
${monHtml}
|
||
<div style="height:8px"></div>
|
||
<div class="vv-wd-sec">Required</div>
|
||
${reqHtml}
|
||
</div>
|
||
<div>
|
||
<div class="vv-wd-sec">Pause at medium pressure</div>
|
||
${pausHtml}
|
||
<div style="height:8px"></div>
|
||
<div class="vv-wd-sec">Stop at hard pressure</div>
|
||
${stopHtml}
|
||
<div style="height:8px"></div>
|
||
<div class="vv-wd-sec">Scan ignore</div>
|
||
${ignHtml}
|
||
</div>
|
||
</div>
|
||
</div>`;
|
||
}
|
||
|
||
// ── Storage watchdog card ─────────────────────────────────────────────────────
|
||
function _storageCard(node, cfg) {
|
||
const st = node.states;
|
||
const nodeCfg = node.config || {};
|
||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Storage Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||
|
||
const storWd = st.storage_wd || {};
|
||
const growthStr = Object.entries(storWd.growth_strikes || {});
|
||
const logStr = Object.entries(storWd.log_strikes || {});
|
||
const totalIssues = growthStr.length + logStr.length;
|
||
const allClear = totalIssues === 0;
|
||
|
||
// Baseline info
|
||
const bCount = storWd.baseline_count ?? 0;
|
||
const bAge = storWd.baseline_age_sec;
|
||
let baselineNote = bCount > 0
|
||
? `${bCount} containers tracked`
|
||
: 'no baseline yet (builds after first cycle)';
|
||
if (bAge != null && bCount > 0) {
|
||
const bAgeStr = bAge < 120 ? bAge + 's ago' : bAge < 3600 ? Math.floor(bAge/60) + 'm ago' : Math.floor(bAge/3600) + 'h ago';
|
||
baselineNote += ` · updated ${bAgeStr}`;
|
||
}
|
||
|
||
// Suppress ceilings configured for this host
|
||
const sizes = Object.entries(nodeCfg.appdata_sizes || {});
|
||
const sizesHtml = sizes.length
|
||
? sizes.map(([c, mb]) => _pill(c + ' <' + Math.round(mb/1024) + 'GB', '')).join('')
|
||
: '';
|
||
|
||
let growthHtml = '';
|
||
if (growthStr.length === 0) {
|
||
growthHtml = '<div style="color:#333;font-size:11px;">no active strikes</div>';
|
||
} else {
|
||
growthHtml = growthStr.map(([name, cnt]) =>
|
||
`<div class="vv-wd-strike-row">
|
||
<span class="vv-wd-strike-name">${vvEscHtml(name)}</span>
|
||
<span class="vv-wd-strike-cnt">${cnt} / ${cfg.stor_strike_lim}</span>
|
||
${_why(name + ' appdata growth', name + ' has ' + cnt
|
||
+ ' growth strike(s) — its appdata is growing more than '
|
||
+ cfg.growth_gb + 'GB per cycle. What is filling up?')}
|
||
</div>`
|
||
).join('');
|
||
}
|
||
|
||
let logHtml = '';
|
||
if (logStr.length === 0) {
|
||
logHtml = '<div style="color:#333;font-size:11px;">no active strikes</div>';
|
||
} else {
|
||
logHtml = logStr.map(([key, cnt]) => {
|
||
const display = key.length > 36 ? '…' + key.slice(-36) : key;
|
||
return `<div class="vv-wd-strike-row">
|
||
<span class="vv-wd-strike-name" style="font-size:10px;" title="${vvEscAttr(key)}">${vvEscHtml(display)}</span>
|
||
<span class="vv-wd-strike-cnt">${cnt} / ${cfg.stor_strike_lim}</span>
|
||
${_why('oversized log ' + key, key + ' has ' + cnt
|
||
+ ' log-size strike(s) against a ' + cfg.log_max_gb
|
||
+ 'GB ceiling. What is writing to it?')}
|
||
</div>`;
|
||
}).join('');
|
||
}
|
||
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-sec">Storage Watchdog</div>
|
||
<div style="display:flex;gap:5px;flex-wrap:wrap;margin-bottom:6px;">
|
||
${allClear ? _pill('all clear', 'ok') : _pill(totalIssues + ' active strike' + (totalIssues !== 1 ? 's' : ''), 'warn')}
|
||
${_pill('growth >' + cfg.growth_gb + 'GB/cycle', '')}
|
||
${_pill('log max ' + cfg.log_max_gb + 'GB', '')}
|
||
${cfg.truncate_logs ? _pill('auto-truncate on', 'ok') : _pill('auto-truncate off', '')}
|
||
</div>
|
||
<div style="font-size:10px;color:#3a3a3a;margin-bottom:8px;">${baselineNote}</div>
|
||
<div style="display:grid;grid-template-columns:1fr 1fr;gap:12px;">
|
||
<div>
|
||
<div class="vv-wd-sec">Growth strikes</div>
|
||
${growthHtml}
|
||
</div>
|
||
<div>
|
||
<div class="vv-wd-sec">Log size strikes</div>
|
||
${logHtml}
|
||
</div>
|
||
</div>
|
||
${sizes.length ? `<hr class="vv-wd-sep"><div class="vv-wd-sec">Suppress ceilings (this host)</div><div class="vv-wd-pill-row">${sizesHtml}</div>` : ''}
|
||
</div>`;
|
||
}
|
||
|
||
// ── Network watchdog card ─────────────────────────────────────────────────────
|
||
function _networkCard(node, cfg) {
|
||
const st = node.states;
|
||
const nodeCfg = node.config || {};
|
||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Network Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||
|
||
const netWd = st.network_wd || {};
|
||
const npmStr = netWd.npm_strikes ?? 0;
|
||
const npmCls = npmStr >= cfg.npm_strike_lim ? 'err' : npmStr > 0 ? 'warn' : 'ok';
|
||
const ddnsDomain = nodeCfg.ddns_domain || '';
|
||
const ddnsCtr = nodeCfg.ddns_container || '';
|
||
const npmUrl = nodeCfg.npm_url || '';
|
||
|
||
const ddnsHtml = ddnsDomain
|
||
? `${_row('DDNS domain', `<span style="color:#888;">${vvEscHtml(ddnsDomain)}</span>`)}
|
||
${ddnsCtr ? _row('DDNS container', `<span style="color:#888;">${vvEscHtml(ddnsCtr)}</span>`) : ''}`
|
||
: _row('DDNS', '<span style="color:#444;">not configured for this host</span>');
|
||
|
||
const npmHtml = npmUrl
|
||
? `${_row('NPM URL', `<span style="color:#888;font-size:10px;">${vvEscHtml(npmUrl)}</span>`)}
|
||
${_row('NPM strikes', `<span class="vv-wd-pill ${npmCls}" style="font-size:10px;">${npmStr} / ${cfg.npm_strike_lim}</span>`
|
||
+ (npmStr > 0 ? ' ' + _why('NPM strikes on ' + node.id,
|
||
'The network watchdog has ' + npmStr + ' strike(s) against NPM on '
|
||
+ node.id + '. What check is failing?') : ''))}`
|
||
: _row('NPM check', '<span style="color:#444;">not configured for this host</span>');
|
||
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-sec">Network Watchdog</div>
|
||
<div style="display:flex;gap:5px;flex-wrap:wrap;margin-bottom:8px;">
|
||
${cfg.net_wd_enabled ? _pill('enabled', 'ok') : _pill('disabled', '')}
|
||
${_pill('Tailscale check ' + (cfg.ts_check ? 'on' : 'off'), cfg.ts_check ? '' : '')}
|
||
${npmUrl ? _pill('NPM ' + npmStr + '/' + cfg.npm_strike_lim + ' strikes', npmCls) : ''}
|
||
</div>
|
||
${ddnsHtml}
|
||
<hr class="vv-wd-sep">
|
||
${npmHtml}
|
||
</div>`;
|
||
}
|
||
|
||
// ── Main render ───────────────────────────────────────────────────────────────
|
||
function _render(data) {
|
||
const nodes = data.nodes || [];
|
||
const cfg = data.cfg || {};
|
||
let html = '';
|
||
|
||
// Per-host: pressure alert (if active) then all 5 watchdog cards in one equal-spaced row
|
||
for (const node of nodes) {
|
||
html += _pressureCard(node);
|
||
html += `<div class="vv-wd-host-row">
|
||
${_systemCard(node, cfg)}
|
||
${_dockerCard(node, cfg)}
|
||
${_stabilityCard(node, cfg)}
|
||
${_storageCard(node, cfg)}
|
||
${_networkCard(node, cfg)}
|
||
</div>`;
|
||
}
|
||
|
||
// Config inventory — separate row, each node span 4 (2 nodes = full row)
|
||
for (const node of nodes) html += _configCard(node);
|
||
|
||
if (!html) html = '<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">No nodes configured.</div>';
|
||
|
||
document.getElementById('vv-wd-grid').innerHTML = html;
|
||
|
||
const ts = data.ts
|
||
? new Date(data.ts * 1000).toLocaleString([], {
|
||
month:'numeric', day:'numeric', year:'numeric',
|
||
hour:'2-digit', minute:'2-digit', second:'2-digit'})
|
||
: '';
|
||
document.getElementById('vv-wd-ts').textContent = ts ? 'Updated: ' + ts : '';
|
||
}
|
||
|
||
function vvWdLoad() {
|
||
fetch('/plugins/varaverk/api/watchdog.php')
|
||
.then(r => r.json())
|
||
.then(_render)
|
||
.catch(() => {
|
||
document.getElementById('vv-wd-grid').innerHTML =
|
||
'<div style="grid-column:1/-1;color:#555;font-size:12px;padding:24px 0;text-align:center;">Error loading watchdog data — check API</div>';
|
||
});
|
||
}
|
||
|
||
// One listener on the grid rather than a handler per button. The grid is replaced wholesale every
|
||
// 30 seconds, so anything bound to a button inside it would be thrown away on the next poll — and
|
||
// re-binding after each render is the version of this that quietly stops working the day a card
|
||
// gains a button and nobody updates the binder.
|
||
document.getElementById('vv-wd-grid').addEventListener('click', ev => {
|
||
const b = ev.target.closest('.vv-why-btn');
|
||
if (!b) return;
|
||
vvWdWhy(b.dataset.label || 'this watchdog', b.dataset.q || 'What does this mean?');
|
||
});
|
||
|
||
// Before the first load, not after. _why() renders nothing while vvWdChat is null, so a render
|
||
// that beat the instance into existence would draw the whole page without a single Why? button
|
||
// and not correct itself until the next poll thirty seconds later.
|
||
if (document.getElementById('vv-wd-ai-chat')) {
|
||
vvWdChat = VvAiChat({
|
||
prefix: 'vv-wd-ai',
|
||
profile: 'varaverk',
|
||
scopeLabel: 'Watchdog',
|
||
// Read at send time, not captured: a Why? moves the scope to the orchestrator's log, and the
|
||
// question is sent from that same click.
|
||
scope: () => vvWdScope,
|
||
// Same reason the Monitor card pins its own: without it the card resumes whatever thread was
|
||
// last touched anywhere, so opening this tab could land it mid-conversation from the Scheduler
|
||
// under a profile this page never offers.
|
||
resumeProfile: 'varaverk',
|
||
// Reasoning is worth watching for a diagnosis and noise for a definition — same rule the
|
||
// Scheduler applies.
|
||
think: p => p === 'troubleshoot',
|
||
empty: 'Ask about a strike, a restart, or a threshold on this page. '
|
||
+ 'Why? on any strike asks against the orchestrator\'s log.',
|
||
});
|
||
}
|
||
|
||
vvWdLoad();
|
||
setInterval(vvWdLoad, 30000);
|
||
|
||
// ── Editable container lists ──────────────────────────────────────────────────
|
||
// Structured rows over the same guarded write path the settings card uses: the value posted is the
|
||
// array *body*, and vv_conf_write_file() splices it between the parens it already found. Nothing
|
||
// here writes conf directly, so the lock, the backup, the `bash -n` check, the read-back and the
|
||
// rollback all still apply exactly as they do to a typed edit.
|
||
//
|
||
// Comments are carried per row rather than regenerated. Every one of these entries has a reason
|
||
// written beside it and rebuilding the array from names and numbers alone would erase all of them
|
||
// in one save — the sort of loss that is invisible until the day someone asks why Emby is 24GB.
|
||
(function () {
|
||
const HOST = document.getElementById('vv-wd-lists');
|
||
if (!HOST || !window.VvWdLists) return;
|
||
const API = '/plugins/varaverk/api/confform.php';
|
||
|
||
const INDENT = ' ';
|
||
// ["Name"]=1234 # why — value may be quoted or bare, and the reason may be absent
|
||
const MAP_RE = /^\s*\["([^"]+)"\]\s*=\s*"?([^"#]*?)"?\s*(?:#\s*(.*))?$/;
|
||
// "Name" # why
|
||
const LIST_RE = /^\s*"([^"]+)"\s*(?:#\s*(.*))?$/;
|
||
|
||
function parseBody(body, kind) {
|
||
const out = [];
|
||
for (const raw of String(body || '').split('\n')) {
|
||
if (!raw.trim() || raw.trim().startsWith('#')) continue;
|
||
const m = raw.match(kind === 'map' ? MAP_RE : LIST_RE);
|
||
if (!m) continue;
|
||
out.push(kind === 'map'
|
||
? { name: m[1], value: (m[2] || '').trim(), note: (m[3] || '').trim() }
|
||
: { name: m[1], value: '', note: (m[2] || '').trim() });
|
||
}
|
||
return out;
|
||
}
|
||
|
||
function buildBody(rows, kind) {
|
||
return rows.map(r => {
|
||
const head = kind === 'map' ? `["${r.name}"]=${r.value}` : `"${r.name}"`;
|
||
return INDENT + (r.note ? head + ' # ' + r.note : head);
|
||
}).join('\n');
|
||
}
|
||
|
||
VvWdLists.forEach((def, idx) => {
|
||
const card = document.createElement('div');
|
||
card.className = 'vv-wl-card';
|
||
HOST.appendChild(card);
|
||
|
||
let rows = parseBody(def.body, def.kind);
|
||
const original = buildBody(rows, def.kind);
|
||
|
||
function draw() {
|
||
card.innerHTML = '';
|
||
const h = document.createElement('div');
|
||
h.className = 'vv-wl-h';
|
||
h.innerHTML = `<span class="vv-wl-t">${vvEscHtml(def.title)}</span>`
|
||
+ `<span class="vv-wl-n">${rows.filter(r => !r.gone).length}</span>`;
|
||
card.appendChild(h);
|
||
|
||
const hint = document.createElement('div');
|
||
hint.className = 'vv-wl-hint';
|
||
hint.textContent = def.hint;
|
||
card.appendChild(hint);
|
||
|
||
rows.forEach((r, i) => {
|
||
const el = document.createElement('div');
|
||
el.className = 'vv-wl-row' + (r.gone ? ' gone' : '');
|
||
const gb = def.kind === 'map' && +r.value > 0
|
||
? `<span class="vv-wl-gb">${(+r.value / 1024).toFixed(1)} GB</span>` : '';
|
||
el.innerHTML =
|
||
`<input class="vv-wl-name" value="${vvEscAttr(r.name)}" spellcheck="false">`
|
||
+ `<span class="vv-wl-gone-t" style="display:none">not installed</span>`
|
||
+ (def.kind === 'map'
|
||
? `<input class="vv-wl-val" value="${vvEscAttr(r.value)}" inputmode="numeric">`
|
||
+ `<span class="vv-wl-unit">${vvEscHtml(def.unit)}</span>${gb}`
|
||
: '')
|
||
+ `<button class="vv-wl-x" title="${r.gone ? 'Keep this entry' : 'Remove this entry'}">`
|
||
+ (r.gone ? '↺' : '×') + '</button>';
|
||
|
||
const [nameEl, valEl] = [el.querySelector('.vv-wl-name'), el.querySelector('.vv-wl-val')];
|
||
const tagEl = el.querySelector('.vv-wl-gone-t');
|
||
// Re-checked on every keystroke, so correcting a stale name clears the badge as soon as it
|
||
// matches something real — which is the whole workflow this is meant to support. Blank
|
||
// rows are exempt: a half-typed new entry is not a dead one.
|
||
const tag = () => {
|
||
const n = nameEl.value.trim();
|
||
tagEl.style.display = (def.check && n !== '' && !_known(n)) ? '' : 'none';
|
||
};
|
||
tag();
|
||
nameEl.addEventListener('input', () => { r.name = nameEl.value.trim(); tag(); mark(); });
|
||
if (valEl) valEl.addEventListener('input', () => { r.value = valEl.value.trim(); mark(); });
|
||
// Marked rather than spliced out, so a mis-click is one click back and the reason written
|
||
// beside the entry is still there to read while deciding.
|
||
el.querySelector('.vv-wl-x').addEventListener('click', () => {
|
||
rows[i].gone = !rows[i].gone; draw();
|
||
});
|
||
card.appendChild(el);
|
||
if (r.note) {
|
||
const n = document.createElement('div');
|
||
n.className = 'vv-wl-note';
|
||
n.textContent = r.note;
|
||
card.appendChild(n);
|
||
}
|
||
});
|
||
|
||
const foot = document.createElement('div');
|
||
foot.className = 'vv-wl-foot';
|
||
foot.innerHTML = `<button class="vv-btn-sm" data-a="add">+ Add</button>`
|
||
+ `<span class="vv-wl-sum"></span>`
|
||
+ `<button class="vv-btn-sm" data-a="save" disabled>Save</button>`;
|
||
card.appendChild(foot);
|
||
foot.querySelector('[data-a="add"]').addEventListener('click', () => {
|
||
rows.push({ name: '', value: def.kind === 'map' ? '1024' : '', note: '' });
|
||
draw();
|
||
const inputs = card.querySelectorAll('.vv-wl-name');
|
||
if (inputs.length) inputs[inputs.length - 1].focus();
|
||
});
|
||
foot.querySelector('[data-a="save"]').addEventListener('click', save);
|
||
mark();
|
||
}
|
||
|
||
function live() { return rows.filter(r => !r.gone && r.name !== ''); }
|
||
|
||
function mark() {
|
||
const changed = buildBody(live(), def.kind) !== original;
|
||
const btn = card.querySelector('[data-a="save"]');
|
||
const sum = card.querySelector('.vv-wl-sum');
|
||
if (btn) btn.disabled = !changed;
|
||
if (sum && !sum.classList.contains('ok') && !sum.classList.contains('err'))
|
||
sum.textContent = changed ? 'unsaved' : '';
|
||
if (sum && changed) { sum.className = 'vv-wl-sum'; sum.textContent = 'unsaved'; }
|
||
}
|
||
|
||
function save() {
|
||
const keep = live();
|
||
const sum = card.querySelector('.vv-wl-sum');
|
||
const btn = card.querySelector('[data-a="save"]');
|
||
// Names are what get written between quotes in a bash array, so anything that could close
|
||
// one early or start a substitution is refused here rather than sent and rejected downstream.
|
||
const bad = keep.find(r => !/^[A-Za-z0-9._-]+$/.test(r.name))
|
||
|| (def.kind === 'map' ? keep.find(r => !/^\d+$/.test(r.value)) : null);
|
||
if (bad) {
|
||
sum.className = 'vv-wl-sum err';
|
||
sum.textContent = 'check "' + bad.name + '"';
|
||
return;
|
||
}
|
||
const dup = keep.map(r => r.name).find((n, i, a) => a.indexOf(n) !== i);
|
||
if (dup) { sum.className = 'vv-wl-sum err'; sum.textContent = dup + ' listed twice'; return; }
|
||
|
||
btn.disabled = true;
|
||
sum.className = 'vv-wl-sum';
|
||
sum.textContent = 'saving…';
|
||
// URLSearchParams, never FormData — a multipart POST to this plugin's endpoints hangs with
|
||
// no status ever returned.
|
||
fetch(API, { method: 'POST',
|
||
headers: { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' },
|
||
body: new URLSearchParams({ changes: JSON.stringify([{
|
||
file: def.file, key: def.key, type: def.type, value: buildBody(keep, def.kind),
|
||
}]) }) })
|
||
.then(r => r.text())
|
||
.then(t => {
|
||
let d;
|
||
try { d = JSON.parse(t); }
|
||
catch (_) {
|
||
throw new Error(t.trim() ? 'unparseable response'
|
||
: 'empty response — rejected before the endpoint ran');
|
||
}
|
||
if (!d.ok) throw new Error(d.error || 'save failed');
|
||
rows = keep.map(r => ({ ...r }));
|
||
def.body = buildBody(rows, def.kind);
|
||
draw();
|
||
const s = card.querySelector('.vv-wl-sum');
|
||
s.className = 'vv-wl-sum ok';
|
||
s.textContent = 'saved — takes effect next cycle';
|
||
})
|
||
.catch(e => {
|
||
const s = card.querySelector('.vv-wl-sum');
|
||
s.className = 'vv-wl-sum err';
|
||
s.textContent = String(e.message || e);
|
||
card.querySelector('[data-a="save"]').disabled = false;
|
||
});
|
||
}
|
||
|
||
draw();
|
||
});
|
||
})();
|
||
|
||
})();
|
||
</script>
|