Gmer4Lfe
8808cbfc04
The page states what struck but never why, and the log that would say is the orchestrator's, not one per watchdog. Why? opens troubleshoot against it — read-only, because the page's contract is that it changes nothing.
692 lines
33 KiB
PHP
692 lines
33 KiB
PHP
<?php
|
||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||
// PURPOSE
|
||
// Watchdog tab. Shows what every watchdog has recorded across all nodes — strike counts,
|
||
// action levels, paused and stopped containers, restart history, OOM and reboot counts.
|
||
//
|
||
// DESIGN PRINCIPLES
|
||
// Pure view. No PHP logic and no include/ dependency — the whole page is markup plus a
|
||
// poll against api/watchdog.php. All parsing lives in include/watchdog.php behind that
|
||
// endpoint, so the page cannot hold a second opinion about what a strike means.
|
||
//
|
||
// Shows every node, not just this one. A watchdog page that only covered the host you
|
||
// happened to open it on would miss exactly the node in trouble.
|
||
//
|
||
// OPERATIONAL SAFEGUARDS
|
||
// Strictly read-only. There is no control on this page that clears a strike, restarts a
|
||
// container, or resets a counter — those belong to the watchdog that owns the state.
|
||
//
|
||
// That holds for the assistant too, and is why the Why? buttons open troubleshoot rather than
|
||
// repair. troubleshoot may read the machine and the orchestrator's log; it holds no conf_write
|
||
// and no way to act. A page whose stated contract is "changes nothing" must not carry the one
|
||
// profile that can — see include/ai_profiles.php for why that grant is deliberately narrow.
|
||
//
|
||
// A node that fails to report renders as unavailable rather than healthy. Absence of data
|
||
// is never drawn as an all-clear.
|
||
//
|
||
// RENDERS
|
||
// Per-node cards: resource / docker / system / storage / network watchdog state,
|
||
// strike pills, restart history, reboot and OOM counts
|
||
// Assistant card, scoped to the Watchdog tab
|
||
// Watchdog conf sections, via the shared settings renderer
|
||
//
|
||
// DEPENDS ON
|
||
// api/watchdog.php polled every 30s → include/watchdog.php
|
||
// include/ai_chat.php → the shared chat component
|
||
require_once dirname(__DIR__) . '/include/confui.php';
|
||
require_once dirname(__DIR__) . '/include/ai_chat.php';
|
||
?>
|
||
<style>
|
||
.vv-wd-card { background:#161616;border:1px solid #2a2a2a;border-radius:6px;padding:10px;min-width:0; }
|
||
.vv-wd-sec { font-size:10px;font-weight:bold;color:#444;letter-spacing:.07em;text-transform:uppercase;margin-bottom:6px; }
|
||
.vv-wd-row { display:flex;justify-content:space-between;align-items:baseline;gap:6px;margin:2px 0; }
|
||
.vv-wd-lbl { font-size:11px;color:#444;white-space:nowrap; }
|
||
.vv-wd-val { font-size:12px;color:#bbb;text-align:right; }
|
||
.vv-wd-sep { border:none;border-top:1px solid #1e1e1e;margin:6px 0; }
|
||
.vv-wd-pill { font-size:10px;padding:1px 6px;border-radius:2px;background:#1e1e1e;color:#666;border:1px solid #272727; }
|
||
.vv-wd-pill.ok { background:#0d1f0d;color:#4caf50;border-color:#1a3a1a; }
|
||
.vv-wd-pill.warn { background:#1f1500;color:#ffb74d;border-color:#3a2800; }
|
||
.vv-wd-pill.err { background:#200d0d;color:#ef5350;border-color:#3a1a1a; }
|
||
.vv-wd-pill-row { display:flex;flex-wrap:wrap;gap:4px;margin-top:4px; }
|
||
.vv-wd-bar { height:5px;border-radius:2px;background:#1e1e1e;margin-top:3px;overflow:hidden; }
|
||
.vv-wd-bar-fill{ height:100%;border-radius:2px;transition:width .3s; }
|
||
.vv-wd-badge { font-size:11px;font-weight:bold;padding:2px 8px;border-radius:3px; }
|
||
.vv-wd-badge.ok { background:#0d1f0d;color:#4caf50; }
|
||
.vv-wd-badge.soft { background:#1f1f00;color:#cddc39; }
|
||
.vv-wd-badge.med { background:#1f1000;color:#ffb74d; }
|
||
.vv-wd-badge.hard { background:#200d0d;color:#ef5350; }
|
||
.vv-wd-node-h { display:flex;align-items:center;gap:8px;margin-bottom:10px; }
|
||
.vv-wd-node-id { font-size:12px;font-weight:bold;color:#666;letter-spacing:.06em;text-transform:uppercase; }
|
||
.vv-wd-dot { width:6px;height:6px;border-radius:50%;flex-shrink:0; }
|
||
.vv-wd-strike-row { display:flex;align-items:baseline;gap:6px;margin:2px 0; }
|
||
.vv-wd-strike-name{ font-size:11px;color:#777;flex:1; }
|
||
.vv-wd-strike-cnt { font-size:11px;font-weight:bold;color:#ffb74d; }
|
||
.vv-wd-reboot-ts { font-size:11px;color:#555;margin:1px 0; }
|
||
.vv-wd-ctr-row { display:flex;justify-content:space-between;align-items:baseline;margin:2px 0; }
|
||
.vv-wd-ctr-name{ font-size:11px;color:#888; }
|
||
.vv-wd-ctr-lim { font-size:11px;color:#555; }
|
||
.vv-wd-pressure{ grid-column:1/-1;border-color:#3a2000;background:#1a1000; }
|
||
|
||
/* One host per row — inner grid sizes all cards equally */
|
||
.vv-wd-host-row {
|
||
grid-column: 1 / -1;
|
||
display: grid;
|
||
grid-template-columns: repeat(5, 1fr);
|
||
gap: 12px;
|
||
}
|
||
@media (max-width: 900px) {
|
||
.vv-wd-host-row { grid-template-columns: 1fr; }
|
||
}
|
||
</style>
|
||
|
||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;padding:0 2px;">
|
||
<span style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;">Watchdog</span>
|
||
<span style="font-size:11px;color:#3a3a3a;" id="vv-wd-ts"></span>
|
||
</div>
|
||
|
||
<div id="vv-wd-grid" style="display:grid;grid-template-columns:repeat(8,1fr);gap:12px;">
|
||
<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">Loading…</div>
|
||
</div>
|
||
|
||
|
||
<?php if (vv_ai_ui_on()): ?>
|
||
<div class="vv-card" id="vv-wd-ai-card" style="margin-top:12px;">
|
||
<?php
|
||
// All three, in this order, before the markup. vv_ai_chat_markup() emits only the boxes — the
|
||
// factory that brings them to life, the profile registry the picker draws from, and the store
|
||
// that keeps a thread across a reload are separate and none of them is implied by the others.
|
||
// Omitting them renders a chat that looks complete and dies on the first click with VvAiChat
|
||
// undefined, which is how the Scheduler's panel behaved before it was given the same three.
|
||
// Each is guarded internally, so calling them here costs nothing on a page that already has.
|
||
vv_ai_profiles_script();
|
||
vv_ai_chat_store_script();
|
||
vv_ai_chat_assets();
|
||
// Reads the page rather than sitting beside it. The scope is sent with every question, and the
|
||
// worker turns it into "they have Watchdog open — read bare references against it", so "why did
|
||
// it restart that" resolves here instead of asking which of forty containers is meant.
|
||
//
|
||
// Starts on the Varaverk assistant, not General Chat as the Monitor card does. The two pages ask
|
||
// different things: the dashboard is where an idle question gets typed, this is a page you only
|
||
// open because something struck. Answers about strikes and thresholds should come from this
|
||
// installation's own docs and state, with sources, rather than from the model's general
|
||
// impression of what a watchdog usually does.
|
||
//
|
||
// No 'height' triple — this card is in normal page flow, not a fixed panel, so the component's
|
||
// own defaults apply and the expand control still works against them. The Scheduler is the only
|
||
// placement whose size is a share of something and therefore has to be told.
|
||
vv_ai_chat_markup('vv-wd-ai', [
|
||
'profile' => 'varaverk',
|
||
'compact' => true,
|
||
'title' => 'Assistant',
|
||
'scopeLabel' => 'Watchdog',
|
||
'empty' => 'Ask about a strike, a restart, or a threshold on this page. '
|
||
. 'Why? on any strike asks against the orchestrator\'s log.',
|
||
'placeholder' => 'Ask about what is on this page…',
|
||
]); ?>
|
||
</div>
|
||
<?php endif; ?>
|
||
|
||
<?php
|
||
// The conf sections this page is about, drawn by the shared renderer. They were reachable
|
||
// only from the Settings tab's catch-all, which is a long way to go for a setting named
|
||
// after the page you are already looking at.
|
||
vv_conf_ui_card('vv-cf-watchdog', 'watchdog', 'Watchdog settings');
|
||
?>
|
||
<script>
|
||
(function() {
|
||
|
||
const GB = 1073741824;
|
||
|
||
// The assistant instance, once the card is on the page; null whenever AI is off. Every Why?
|
||
// button checks it, and none of them render without it — a control that opens a panel that does
|
||
// not exist is worse than no control.
|
||
let vvWdChat = null;
|
||
// What the chat says the operator is looking at. Starts as the tab itself and moves to the
|
||
// orchestrator's log when a Why? is pressed, because that is the only place any of this is
|
||
// written down: the watchdogs log through watchdog_orchestrator.sh and have no log files of
|
||
// their own — checked against /var/log/varaverk, where Watchdogs/ does not exist.
|
||
let vvWdScope = 'Watchdog';
|
||
const WD_LOG = 'Orchestrators/watchdog_orchestrator';
|
||
|
||
// Rendered as data attributes and read back by one delegated listener rather than written into an
|
||
// onclick. Container names and log paths reach these buttons from conf and from the filesystem, and
|
||
// a name containing a quote closes the attribute early and takes the rest of the handler with it.
|
||
// vvEscAttr is the one that escapes quotes; vvEscHtml deliberately does not.
|
||
function _why(label, q) {
|
||
if (!vvWdChat) return '';
|
||
return `<button class="vv-btn-sm vv-why-btn" title="Ask the assistant about this"
|
||
data-label="${vvEscAttr(label)}" data-q="${vvEscAttr(q)}">Why?</button>`;
|
||
}
|
||
|
||
function vvWdWhy(label, q) {
|
||
if (!vvWdChat || vvWdChat.busy()) return;
|
||
// Troubleshoot, never repair. This page changes nothing, and repair is the one profile that can
|
||
// — see the safeguard note in the header.
|
||
vvWdScope = WD_LOG;
|
||
vvWdChat.retarget('troubleshoot', label, 'now looking at ' + label);
|
||
const input = document.getElementById('vv-wd-ai-input');
|
||
if (input) input.value = q;
|
||
vvWdChat.send();
|
||
const card = document.getElementById('vv-wd-ai-card');
|
||
if (card) card.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
|
||
}
|
||
|
||
function _fmtBytes(b) {
|
||
if (!b) return '0';
|
||
if (b >= GB) return (b / GB).toFixed(1) + 'G';
|
||
if (b >= 1048576) return (b / 1048576).toFixed(0) + 'M';
|
||
return (b / 1024).toFixed(0) + 'K';
|
||
}
|
||
|
||
function _relTime(ts) {
|
||
if (!ts) return '—';
|
||
const s = Math.floor(Date.now() / 1000) - ts;
|
||
if (s < 60) return 'just now';
|
||
if (s < 3600) return Math.floor(s / 60) + 'm ago';
|
||
if (s < 86400) return Math.floor(s / 3600) + 'h ago';
|
||
return Math.floor(s / 86400) + 'd ago';
|
||
}
|
||
|
||
function _dur(s) {
|
||
const d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60);
|
||
if (d) return d + 'd ' + h + 'h';
|
||
if (h) return h + 'h ' + m + 'm';
|
||
return m + 'm';
|
||
}
|
||
|
||
function _row(lbl, val) {
|
||
return `<div class="vv-wd-row"><span class="vv-wd-lbl">${lbl}</span><span class="vv-wd-val">${val}</span></div>`;
|
||
}
|
||
|
||
function _bar(pct, col) {
|
||
return `<div class="vv-wd-bar"><div class="vv-wd-bar-fill" style="width:${Math.min(pct,100)}%;background:${col}"></div></div>`;
|
||
}
|
||
|
||
// Escapes its own label, so every caller passes plain text and none of them has to remember. The
|
||
// labels here are container names, log paths and DDNS domains — all of them values this page reads
|
||
// rather than writes, which is exactly the category that must not be trusted into markup.
|
||
function _pill(label, cls) {
|
||
return `<span class="vv-wd-pill ${cls}">${vvEscHtml(label)}</span>`;
|
||
}
|
||
|
||
function _levelLabel(level) {
|
||
return ['OK', 'SOFT', 'MEDIUM', 'HARD'][level] || '?';
|
||
}
|
||
function _levelCls(level) {
|
||
return ['ok', 'soft', 'med', 'hard'][level] || 'ok';
|
||
}
|
||
|
||
// ── Pressure alert card ───────────────────────────────────────────────────────
|
||
function _pressureCard(node) {
|
||
const st = node.states;
|
||
if (!st || st.rw_level === 0) return '';
|
||
const level = st.rw_level;
|
||
const cls = _levelCls(level);
|
||
const label = _levelLabel(level);
|
||
|
||
const paused = (st.rw_paused || []).filter(Boolean);
|
||
const stopped = (st.rw_stopped || []).filter(Boolean);
|
||
const pausedHtml = paused.length ? paused.map(c => _pill(c, 'warn')).join('') : '';
|
||
const stoppedHtml = stopped.length ? stopped.map(c => _pill(c, 'err')).join('') : '';
|
||
|
||
return `<div class="vv-wd-card vv-wd-pressure" style="grid-column:1/-1">
|
||
<div style="display:flex;align-items:center;gap:10px;margin-bottom:8px;">
|
||
<span class="vv-wd-badge ${cls}">PRESSURE ${label}</span>
|
||
<span style="font-size:11px;color:#7a5020;">${vvEscHtml(node.id)} (${vvEscHtml(node.hostname)})</span>
|
||
${_why(node.id + ' memory pressure',
|
||
'The resource watchdog is at ' + label + ' pressure on ' + node.id
|
||
+ '. What drove it there, and what did it pause or stop?')}
|
||
${st.mem_shutdown ? `<span class="vv-wd-badge hard" style="margin-left:auto;">MEM SHUTDOWN ACTIVE</span>` : ''}
|
||
</div>
|
||
${paused.length ? `<div style="margin-bottom:4px;"><span class="vv-wd-lbl">Paused:</span> <span class="vv-wd-pill-row" style="display:inline-flex;">${pausedHtml}</span></div>` : ''}
|
||
${stopped.length ? `<div><span class="vv-wd-lbl">Stopped:</span> <span class="vv-wd-pill-row" style="display:inline-flex;">${stoppedHtml}</span></div>` : ''}
|
||
</div>`;
|
||
}
|
||
|
||
// ── System health card ────────────────────────────────────────────────────────
|
||
function _systemCard(node, cfg) {
|
||
const sys = node.system;
|
||
if (!sys) {
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-node-h">
|
||
<span class="vv-wd-dot" style="background:#444"></span>
|
||
<span class="vv-wd-node-id">${vvEscHtml(node.id)}</span>
|
||
<span style="font-size:11px;color:#3a3a3a;">${vvEscHtml(node.hostname)}</span>
|
||
<span class="vv-wd-badge" style="margin-left:auto;background:#1a1a1a;color:#444">UNREACHABLE</span>
|
||
</div>
|
||
</div>`;
|
||
}
|
||
|
||
const memGb = sys.mem_avail / GB;
|
||
const memTotGb = sys.mem_total / GB;
|
||
const usedPct = memTotGb > 0 ? ((memTotGb - memGb) / memTotGb) * 100 : 0;
|
||
const memCol = memGb < cfg.sys_mem_gb ? '#ef5350'
|
||
: memGb < cfg.rw_hard_gb ? '#ef5350'
|
||
: memGb < cfg.rw_medium_gb ? '#ffb74d'
|
||
: memGb < cfg.rw_soft_gb ? '#cddc39' : '#4caf50';
|
||
|
||
const loadPct = sys.cores > 0 ? (sys.load1 / (sys.cores * cfg.rw_load_med)) * 100 : 0;
|
||
const loadCol = sys.load1 > sys.cores * cfg.rw_load_med ? '#ef5350'
|
||
: sys.load1 > sys.cores * cfg.rw_load_soft ? '#ffb74d'
|
||
: '#4caf50';
|
||
|
||
const apiOnly = sys.api_only === true;
|
||
const daemonDot = sys.daemon_ok === null ? '#555' : sys.daemon_ok ? '#4caf50' : '#ef5350';
|
||
const daemonTxt = sys.daemon_ok === null ? '—' : sys.daemon_ok ? 'daemon ok' : 'daemon err';
|
||
|
||
const st = node.states || {};
|
||
const level = st.rw_level || 0;
|
||
const dotCol = level >= 3 ? '#ef5350' : level >= 2 ? '#ffb74d' : level >= 1 ? '#cddc39'
|
||
: apiOnly ? '#4a7a9b' // blue-grey: API-only, no watchdog state
|
||
: '#4caf50';
|
||
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-node-h">
|
||
<span class="vv-wd-dot" style="background:${dotCol}"></span>
|
||
<span class="vv-wd-node-id">${vvEscHtml(node.id)}</span>
|
||
<span style="font-size:11px;color:#3a3a3a;">${vvEscHtml(node.hostname)}</span>
|
||
<span class="vv-wd-badge ${apiOnly ? '' : _levelCls(level)}"
|
||
style="margin-left:auto;${apiOnly ? 'background:#0d1f2a;color:#4a9eff;' : ''}"
|
||
>${apiOnly ? 'API ONLY' : _levelLabel(level)}</span>
|
||
</div>
|
||
<div class="vv-wd-sec">System</div>
|
||
${_row('RAM free', `<span style="color:${memCol}">${_fmtBytes(sys.mem_avail)}</span> / ${_fmtBytes(sys.mem_total)}`)}
|
||
${_bar(usedPct, memCol)}
|
||
<div style="display:flex;justify-content:space-between;margin-top:1px;font-size:10px;color:#333">
|
||
<span>free</span>
|
||
<span>${cfg.rw_soft_gb}G soft · ${cfg.rw_hard_gb}G hard · ${cfg.sys_mem_gb}G reboot</span>
|
||
</div>
|
||
<div style="height:5px"></div>
|
||
${!apiOnly ? `${_row('Load avg', `<span style="color:${loadCol}">${sys.load1.toFixed(2)}</span> / ${sys.cores} cores`)}
|
||
${_bar(loadPct, loadCol)}
|
||
<div style="height:5px"></div>` : ''}
|
||
${_row('Uptime', sys.uptime ? _dur(sys.uptime) : '—')}
|
||
${_row('Docker', `<span style="color:${daemonDot}">${daemonTxt}</span>`)}
|
||
${!apiOnly ? (sys.oom_count > 0 ? _row('OOM kills', `<span style="color:#ef5350">${sys.oom_count}</span>`) : _row('OOM kills', '<span style="color:#333">0</span>')) : ''}
|
||
${apiOnly ? `<div style="font-size:10px;color:#333;margin-top:6px;">Watchdog state unavailable — SSH not configured</div>` : ''}
|
||
</div>`;
|
||
}
|
||
|
||
// ── Docker watchdog state card ────────────────────────────────────────────────
|
||
function _dockerCard(node, cfg) {
|
||
const st = node.states;
|
||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Docker Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||
|
||
const strikes = Object.entries(st.ctr_strikes || {});
|
||
const skiplist = st.skiplist || [];
|
||
const restarts = (st.restarts || []).slice(0, 10);
|
||
const daemonCls = st.daemon_strikes > 0 ? 'err' : 'ok';
|
||
const allOk = strikes.length === 0 && skiplist.length === 0 && st.daemon_strikes === 0;
|
||
|
||
// Group restarts by container for last-24h summary
|
||
const rCounts = {};
|
||
for (const r of restarts) {
|
||
rCounts[r.name] = (rCounts[r.name] || 0) + 1;
|
||
}
|
||
|
||
let strikesHtml = '';
|
||
if (strikes.length === 0 && st.daemon_strikes === 0) {
|
||
strikesHtml = '<div style="color:#333;font-size:11px;">0 active strikes</div>';
|
||
} else {
|
||
if (st.daemon_strikes > 0) {
|
||
strikesHtml += `<div class="vv-wd-strike-row"><span class="vv-wd-strike-name">daemon</span><span class="vv-wd-strike-cnt">${st.daemon_strikes}</span></div>`;
|
||
}
|
||
for (const [name, cnt] of strikes) {
|
||
strikesHtml += `<div class="vv-wd-strike-row">
|
||
<span class="vv-wd-strike-name">${vvEscHtml(name)}</span>
|
||
<span class="vv-wd-strike-cnt">${cnt} / ${cfg.cpu_fail_lim}</span>
|
||
${_why(name + ' strikes', 'The docker watchdog has ' + cnt + ' strike(s) against '
|
||
+ name + '. What is failing its check?')}
|
||
</div>`;
|
||
}
|
||
}
|
||
|
||
let skipHtml = '';
|
||
if (skiplist.length === 0) {
|
||
skipHtml = '<div style="color:#333;font-size:11px;">empty</div>';
|
||
} else {
|
||
skipHtml = `<div class="vv-wd-pill-row">${skiplist.map(c => _pill(c, 'err')).join('')}</div>`;
|
||
}
|
||
|
||
let restartHtml = '';
|
||
const rcEntries = Object.entries(rCounts);
|
||
if (rcEntries.length === 0) {
|
||
restartHtml = '<div style="color:#333;font-size:11px;">none (24h)</div>';
|
||
} else {
|
||
restartHtml = rcEntries.map(([n, c]) =>
|
||
`<div class="vv-wd-row">
|
||
<span class="vv-wd-lbl">${vvEscHtml(n)}</span>
|
||
<span class="vv-wd-val" style="color:${c >= cfg.restart_limit ? '#ef5350' : '#ffb74d'}">${c}×</span>
|
||
${_why(n + ' restarts', n + ' has been restarted ' + c
|
||
+ ' time(s) in the last 24 hours. Why does it keep needing a restart?')}
|
||
</div>`
|
||
).join('');
|
||
}
|
||
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-sec">Docker Watchdog</div>
|
||
<div style="display:flex;gap:6px;margin-bottom:8px;">
|
||
${_pill(st.daemon_restart ? 'daemon restarted' : 'daemon ok', daemonCls)}
|
||
${allOk ? _pill('all clear', 'ok') : ''}
|
||
</div>
|
||
<div class="vv-wd-sec">Strikes</div>
|
||
${strikesHtml}
|
||
<hr class="vv-wd-sep">
|
||
<div class="vv-wd-sec">Skip list</div>
|
||
${skipHtml}
|
||
<hr class="vv-wd-sep">
|
||
<div class="vv-wd-sec">Restarts (24h)</div>
|
||
${restartHtml}
|
||
</div>`;
|
||
}
|
||
|
||
// ── Stability / reboot card ───────────────────────────────────────────────────
|
||
function _stabilityCard(node, cfg) {
|
||
const st = node.states;
|
||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Stability</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||
|
||
const reboots = st.reboots || [];
|
||
const sysStr = Object.entries(st.sys_strikes || {});
|
||
const rebootCls = reboots.length >= cfg.reboot_limit ? 'err' : reboots.length > 0 ? 'warn' : 'ok';
|
||
|
||
let sysHtml = '';
|
||
if (sysStr.length === 0) {
|
||
sysHtml = '<div style="color:#333;font-size:11px;">0 active strikes</div>';
|
||
} else {
|
||
sysHtml = sysStr.map(([k, v]) =>
|
||
`<div class="vv-wd-strike-row">
|
||
<span class="vv-wd-strike-name" style="font-size:10px;">${vvEscHtml(k.replace(/_/g,' '))}</span>
|
||
<span class="vv-wd-strike-cnt">${v}</span>
|
||
${_why(k.replace(/_/g,' ') + ' strikes', 'The stability watchdog has ' + v
|
||
+ ' strike(s) for ' + k.replace(/_/g,' ') + '. What is it detecting?')}
|
||
</div>`
|
||
).join('');
|
||
}
|
||
|
||
// The one entry on this page that is always worth explaining, so the button is on the heading
|
||
// rather than each row — an unattended reboot is a single question, not one per occurrence.
|
||
const rebootHtml = reboots.length === 0
|
||
? '<div style="color:#333;font-size:11px;">none (12h)</div>'
|
||
: reboots.map(ts => `<div class="vv-wd-reboot-ts">${_relTime(ts)}</div>`).join('')
|
||
+ `<div style="margin-top:5px;">${_why(node.id + ' reboots',
|
||
'The stability watchdog rebooted ' + node.id + ' ' + reboots.length
|
||
+ ' time(s) in the last ' + cfg.reboot_window + ' hours. What forced it?')}</div>`;
|
||
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-sec">Stability</div>
|
||
<div style="display:flex;gap:6px;margin-bottom:8px;">
|
||
${_pill(`${reboots.length} / ${cfg.reboot_limit} reboots`, rebootCls)}
|
||
${_pill(`${cfg.reboot_window}h window`, '')}
|
||
</div>
|
||
<div class="vv-wd-sec">Strikes</div>
|
||
${sysHtml}
|
||
<hr class="vv-wd-sep">
|
||
<div class="vv-wd-sec">Reboot history</div>
|
||
${rebootHtml}
|
||
</div>`;
|
||
}
|
||
|
||
// ── Config inventory card ─────────────────────────────────────────────────────
|
||
function _configCard(node) {
|
||
const cfg = node.config || {};
|
||
const mon = Object.entries(cfg.monitored || {});
|
||
const req = cfg.required || [];
|
||
const ign = cfg.ignore || [];
|
||
const paus = cfg.pause_list|| [];
|
||
const stop = cfg.stop_list || [];
|
||
const crit = cfg.critical || [];
|
||
|
||
const monHtml = mon.length === 0
|
||
? '<div style="color:#333;font-size:11px;">none</div>'
|
||
: mon.map(([name, mb]) => {
|
||
const gb = (mb / 1024).toFixed(0);
|
||
return `<div class="vv-wd-ctr-row"><span class="vv-wd-ctr-name">${vvEscHtml(name)}</span><span class="vv-wd-ctr-lim">${gb} GB</span></div>`;
|
||
}).join('');
|
||
|
||
const reqHtml = req.length === 0
|
||
? '<div style="color:#333;font-size:11px;">none</div>'
|
||
: `<div class="vv-wd-pill-row">${req.map(c => _pill(c, crit.includes(c) ? '' : '')).join('')}</div>`;
|
||
|
||
const ignHtml = ign.length === 0
|
||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||
: `<div class="vv-wd-pill-row">${ign.map(c => _pill(c, '')).join('')}</div>`;
|
||
|
||
const pausHtml = paus.length === 0
|
||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||
: `<div class="vv-wd-pill-row">${paus.map(c => _pill(c, 'warn')).join('')}</div>`;
|
||
|
||
const stopHtml = stop.length === 0
|
||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||
: `<div class="vv-wd-pill-row">${stop.map(c => _pill(c, 'err')).join('')}</div>`;
|
||
|
||
return `<div class="vv-wd-card" style="grid-column:span 4;">
|
||
<div style="display:flex;gap:6px;align-items:center;margin-bottom:8px;">
|
||
<span class="vv-wd-node-id">${vvEscHtml(node.id)}</span>
|
||
<span style="font-size:11px;color:#3a3a3a;">${vvEscHtml(node.hostname)}</span>
|
||
</div>
|
||
<div style="display:grid;grid-template-columns:1fr 1fr;gap:10px;">
|
||
<div>
|
||
<div class="vv-wd-sec">Mem limits (Tier 1)</div>
|
||
${monHtml}
|
||
<div style="height:8px"></div>
|
||
<div class="vv-wd-sec">Required</div>
|
||
${reqHtml}
|
||
</div>
|
||
<div>
|
||
<div class="vv-wd-sec">Pause at medium pressure</div>
|
||
${pausHtml}
|
||
<div style="height:8px"></div>
|
||
<div class="vv-wd-sec">Stop at hard pressure</div>
|
||
${stopHtml}
|
||
<div style="height:8px"></div>
|
||
<div class="vv-wd-sec">Scan ignore</div>
|
||
${ignHtml}
|
||
</div>
|
||
</div>
|
||
</div>`;
|
||
}
|
||
|
||
// ── Storage watchdog card ─────────────────────────────────────────────────────
|
||
function _storageCard(node, cfg) {
|
||
const st = node.states;
|
||
const nodeCfg = node.config || {};
|
||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Storage Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||
|
||
const storWd = st.storage_wd || {};
|
||
const growthStr = Object.entries(storWd.growth_strikes || {});
|
||
const logStr = Object.entries(storWd.log_strikes || {});
|
||
const totalIssues = growthStr.length + logStr.length;
|
||
const allClear = totalIssues === 0;
|
||
|
||
// Baseline info
|
||
const bCount = storWd.baseline_count ?? 0;
|
||
const bAge = storWd.baseline_age_sec;
|
||
let baselineNote = bCount > 0
|
||
? `${bCount} containers tracked`
|
||
: 'no baseline yet (builds after first cycle)';
|
||
if (bAge != null && bCount > 0) {
|
||
const bAgeStr = bAge < 120 ? bAge + 's ago' : bAge < 3600 ? Math.floor(bAge/60) + 'm ago' : Math.floor(bAge/3600) + 'h ago';
|
||
baselineNote += ` · updated ${bAgeStr}`;
|
||
}
|
||
|
||
// Suppress ceilings configured for this host
|
||
const sizes = Object.entries(nodeCfg.appdata_sizes || {});
|
||
const sizesHtml = sizes.length
|
||
? sizes.map(([c, mb]) => _pill(c + ' <' + Math.round(mb/1024) + 'GB', '')).join('')
|
||
: '';
|
||
|
||
let growthHtml = '';
|
||
if (growthStr.length === 0) {
|
||
growthHtml = '<div style="color:#333;font-size:11px;">no active strikes</div>';
|
||
} else {
|
||
growthHtml = growthStr.map(([name, cnt]) =>
|
||
`<div class="vv-wd-strike-row">
|
||
<span class="vv-wd-strike-name">${vvEscHtml(name)}</span>
|
||
<span class="vv-wd-strike-cnt">${cnt} / ${cfg.stor_strike_lim}</span>
|
||
${_why(name + ' appdata growth', name + ' has ' + cnt
|
||
+ ' growth strike(s) — its appdata is growing more than '
|
||
+ cfg.growth_gb + 'GB per cycle. What is filling up?')}
|
||
</div>`
|
||
).join('');
|
||
}
|
||
|
||
let logHtml = '';
|
||
if (logStr.length === 0) {
|
||
logHtml = '<div style="color:#333;font-size:11px;">no active strikes</div>';
|
||
} else {
|
||
logHtml = logStr.map(([key, cnt]) => {
|
||
const display = key.length > 36 ? '…' + key.slice(-36) : key;
|
||
return `<div class="vv-wd-strike-row">
|
||
<span class="vv-wd-strike-name" style="font-size:10px;" title="${vvEscAttr(key)}">${vvEscHtml(display)}</span>
|
||
<span class="vv-wd-strike-cnt">${cnt} / ${cfg.stor_strike_lim}</span>
|
||
${_why('oversized log ' + key, key + ' has ' + cnt
|
||
+ ' log-size strike(s) against a ' + cfg.log_max_gb
|
||
+ 'GB ceiling. What is writing to it?')}
|
||
</div>`;
|
||
}).join('');
|
||
}
|
||
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-sec">Storage Watchdog</div>
|
||
<div style="display:flex;gap:5px;flex-wrap:wrap;margin-bottom:6px;">
|
||
${allClear ? _pill('all clear', 'ok') : _pill(totalIssues + ' active strike' + (totalIssues !== 1 ? 's' : ''), 'warn')}
|
||
${_pill('growth >' + cfg.growth_gb + 'GB/cycle', '')}
|
||
${_pill('log max ' + cfg.log_max_gb + 'GB', '')}
|
||
${cfg.truncate_logs ? _pill('auto-truncate on', 'ok') : _pill('auto-truncate off', '')}
|
||
</div>
|
||
<div style="font-size:10px;color:#3a3a3a;margin-bottom:8px;">${baselineNote}</div>
|
||
<div style="display:grid;grid-template-columns:1fr 1fr;gap:12px;">
|
||
<div>
|
||
<div class="vv-wd-sec">Growth strikes</div>
|
||
${growthHtml}
|
||
</div>
|
||
<div>
|
||
<div class="vv-wd-sec">Log size strikes</div>
|
||
${logHtml}
|
||
</div>
|
||
</div>
|
||
${sizes.length ? `<hr class="vv-wd-sep"><div class="vv-wd-sec">Suppress ceilings (this host)</div><div class="vv-wd-pill-row">${sizesHtml}</div>` : ''}
|
||
</div>`;
|
||
}
|
||
|
||
// ── Network watchdog card ─────────────────────────────────────────────────────
|
||
function _networkCard(node, cfg) {
|
||
const st = node.states;
|
||
const nodeCfg = node.config || {};
|
||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Network Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||
|
||
const netWd = st.network_wd || {};
|
||
const npmStr = netWd.npm_strikes ?? 0;
|
||
const npmCls = npmStr >= cfg.npm_strike_lim ? 'err' : npmStr > 0 ? 'warn' : 'ok';
|
||
const ddnsDomain = nodeCfg.ddns_domain || '';
|
||
const ddnsCtr = nodeCfg.ddns_container || '';
|
||
const npmUrl = nodeCfg.npm_url || '';
|
||
|
||
const ddnsHtml = ddnsDomain
|
||
? `${_row('DDNS domain', `<span style="color:#888;">${vvEscHtml(ddnsDomain)}</span>`)}
|
||
${ddnsCtr ? _row('DDNS container', `<span style="color:#888;">${vvEscHtml(ddnsCtr)}</span>`) : ''}`
|
||
: _row('DDNS', '<span style="color:#444;">not configured for this host</span>');
|
||
|
||
const npmHtml = npmUrl
|
||
? `${_row('NPM URL', `<span style="color:#888;font-size:10px;">${vvEscHtml(npmUrl)}</span>`)}
|
||
${_row('NPM strikes', `<span class="vv-wd-pill ${npmCls}" style="font-size:10px;">${npmStr} / ${cfg.npm_strike_lim}</span>`
|
||
+ (npmStr > 0 ? ' ' + _why('NPM strikes on ' + node.id,
|
||
'The network watchdog has ' + npmStr + ' strike(s) against NPM on '
|
||
+ node.id + '. What check is failing?') : ''))}`
|
||
: _row('NPM check', '<span style="color:#444;">not configured for this host</span>');
|
||
|
||
return `<div class="vv-wd-card">
|
||
<div class="vv-wd-sec">Network Watchdog</div>
|
||
<div style="display:flex;gap:5px;flex-wrap:wrap;margin-bottom:8px;">
|
||
${cfg.net_wd_enabled ? _pill('enabled', 'ok') : _pill('disabled', '')}
|
||
${_pill('Tailscale check ' + (cfg.ts_check ? 'on' : 'off'), cfg.ts_check ? '' : '')}
|
||
${npmUrl ? _pill('NPM ' + npmStr + '/' + cfg.npm_strike_lim + ' strikes', npmCls) : ''}
|
||
</div>
|
||
${ddnsHtml}
|
||
<hr class="vv-wd-sep">
|
||
${npmHtml}
|
||
</div>`;
|
||
}
|
||
|
||
// ── Main render ───────────────────────────────────────────────────────────────
|
||
function _render(data) {
|
||
const nodes = data.nodes || [];
|
||
const cfg = data.cfg || {};
|
||
let html = '';
|
||
|
||
// Per-host: pressure alert (if active) then all 5 watchdog cards in one equal-spaced row
|
||
for (const node of nodes) {
|
||
html += _pressureCard(node);
|
||
html += `<div class="vv-wd-host-row">
|
||
${_systemCard(node, cfg)}
|
||
${_dockerCard(node, cfg)}
|
||
${_stabilityCard(node, cfg)}
|
||
${_storageCard(node, cfg)}
|
||
${_networkCard(node, cfg)}
|
||
</div>`;
|
||
}
|
||
|
||
// Config inventory — separate row, each node span 4 (2 nodes = full row)
|
||
for (const node of nodes) html += _configCard(node);
|
||
|
||
if (!html) html = '<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">No nodes configured.</div>';
|
||
|
||
document.getElementById('vv-wd-grid').innerHTML = html;
|
||
|
||
const ts = data.ts
|
||
? new Date(data.ts * 1000).toLocaleString([], {
|
||
month:'numeric', day:'numeric', year:'numeric',
|
||
hour:'2-digit', minute:'2-digit', second:'2-digit'})
|
||
: '';
|
||
document.getElementById('vv-wd-ts').textContent = ts ? 'Updated: ' + ts : '';
|
||
}
|
||
|
||
function vvWdLoad() {
|
||
fetch('/plugins/varaverk/api/watchdog.php')
|
||
.then(r => r.json())
|
||
.then(_render)
|
||
.catch(() => {
|
||
document.getElementById('vv-wd-grid').innerHTML =
|
||
'<div style="grid-column:1/-1;color:#555;font-size:12px;padding:24px 0;text-align:center;">Error loading watchdog data — check API</div>';
|
||
});
|
||
}
|
||
|
||
// One listener on the grid rather than a handler per button. The grid is replaced wholesale every
|
||
// 30 seconds, so anything bound to a button inside it would be thrown away on the next poll — and
|
||
// re-binding after each render is the version of this that quietly stops working the day a card
|
||
// gains a button and nobody updates the binder.
|
||
document.getElementById('vv-wd-grid').addEventListener('click', ev => {
|
||
const b = ev.target.closest('.vv-why-btn');
|
||
if (!b) return;
|
||
vvWdWhy(b.dataset.label || 'this watchdog', b.dataset.q || 'What does this mean?');
|
||
});
|
||
|
||
// Before the first load, not after. _why() renders nothing while vvWdChat is null, so a render
|
||
// that beat the instance into existence would draw the whole page without a single Why? button
|
||
// and not correct itself until the next poll thirty seconds later.
|
||
if (document.getElementById('vv-wd-ai-chat')) {
|
||
vvWdChat = VvAiChat({
|
||
prefix: 'vv-wd-ai',
|
||
profile: 'varaverk',
|
||
scopeLabel: 'Watchdog',
|
||
// Read at send time, not captured: a Why? moves the scope to the orchestrator's log, and the
|
||
// question is sent from that same click.
|
||
scope: () => vvWdScope,
|
||
// Same reason the Monitor card pins its own: without it the card resumes whatever thread was
|
||
// last touched anywhere, so opening this tab could land it mid-conversation from the Scheduler
|
||
// under a profile this page never offers.
|
||
resumeProfile: 'varaverk',
|
||
// Reasoning is worth watching for a diagnosis and noise for a definition — same rule the
|
||
// Scheduler applies.
|
||
think: p => p === 'troubleshoot',
|
||
empty: 'Ask about a strike, a restart, or a threshold on this page. '
|
||
+ 'Why? on any strike asks against the orchestrator\'s log.',
|
||
});
|
||
}
|
||
|
||
vvWdLoad();
|
||
setInterval(vvWdLoad, 30000);
|
||
|
||
})();
|
||
</script>
|