FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Show when a watchdog cycle last completed

Browse Source 
Without it the page renders zeros, all-clear pills and green dots when the
orchestrator has stopped entirely — its healthiest appearance describing a
machine with no watchdogs, against a header promising absence is never drawn as
an all-clear. The heartbeat already existed; overdue is measured against the
cron schedule rather than an assumed fifteen minutes.
This commit is contained in:
Gmer4Lfe
2026-08-14 16:18:14 -04:00
parent a7387c27fb
commit 9f32644c32
2 changed files with 50 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Plugin/unraid/include/watchdog.php
+27
View File
@@ -84,6 +84,25 @@ function vv_wd_installed_containers(): array {
return array_values(array_filter(array_map('trim', $out), fn($n) => $n !== ''));
}
// How many minutes are supposed to pass between watchdog cycles.
//
// Parsed from varaverk.cron, which the scheduler regenerates at array start and is the only place
// the answer exists — the interval is a schedule, not a conf key. Reading it rather than assuming
// 15 means that changing the schedule also moves the point at which the page calls a cycle
// overdue, instead of leaving a constant here to drift out of agreement with reality.
//
// Falls back to 15 when the cron is missing or the entry is written in a form this does not
// recognise. A wrong-but-sane interval produces a slightly early or late warning; refusing to
// answer would remove the liveness check altogether, which is the failure being fixed.
function vv_wd_cycle_interval_min(): int {
$cron = @file_get_contents('/boot/config/plugins/varaverk/varaverk.cron') ?: '';
foreach (explode("\n", $cron) as $line) {
if (!str_contains($line, 'watchdog_orchestrator')) continue;
if (preg_match('#^\s*\*/(\d+)\s#', $line, $m)) return max(1, (int)$m[1]);
}
return 15;
}
// ── Conf array parser (bash arrays) ──────────────────────────────────────────
function vv_wd_bash_array(string $raw, string $varname): array {
@@ -272,6 +291,9 @@ function vv_wd_local_states(string $restartLogPath): array {
'ctr_strikes' => $strikes,
'skiplist' => vv_wd_parse_skiplist($skipRaw),
'sys_strikes' => $sysStrikes,
// The heartbeat, not a strike — see VV_WD_SYS_BOOKKEEPING. stability_watchdog.sh writes it
// last in the chain, so a fresh value means a whole cycle completed rather than started.
'last_cycle' => (int)($sys['watchdog_cycle'] ?? 0),
'reboots' => vv_wd_parse_reboot_log($rebootRaw),
'restarts' => vv_wd_parse_restart_log($restartRaw),
'storage_wd' => vv_wd_parse_storage_state($storRaw) + [
@@ -391,6 +413,7 @@ function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath):
'ctr_strikes' => $strikes,
'skiplist' => vv_wd_parse_skiplist($skipRaw),
'sys_strikes' => $sysStrikes,
'last_cycle' => (int)($sys['watchdog_cycle'] ?? 0),
'reboots' => vv_wd_parse_reboot_log($rebootRaw),
'restarts' => vv_wd_parse_restart_log($restartRaw),
'storage_wd' => vv_wd_parse_storage_state($storRaw) + [
@@ -465,6 +488,10 @@ function vv_wd_all(): array {
// than omit a member of SYSTEM_WATCHDOG_SCRIPTS entirely and let its absence read as fine.
'fallback_on' => vv_wd_scalar($masterRaw, 'FALLBACK_ENABLED') === 'true',
'conf_sync_on' => vv_wd_scalar($masterRaw, 'CONF_SYNC_ENABLED') !== 'false',
// How often a cycle is supposed to happen, so "overdue" is measured against the schedule
// in force rather than a number written here. Read from the cron because that is where
// the answer lives — there is no conf key for it.
'cycle_min' => vv_wd_cycle_interval_min(),
];
// SSH key from current host conf
Plugin/unraid/pages/watchdog.php
+23
View File
@@ -419,6 +419,27 @@ function _systemCard(node, cfg) {
? '<span style="color:#7a6a20;">off — CONF_SYNC_ENABLED</span>'
: '<span style="color:#4caf50;">active</span>';
// ── Is anything still running? ──────────────────────────────────────────────
// Every other number on this card is only meaningful if a cycle actually happened. If the
// orchestrator stopped, the strike maps stop changing and the whole page renders zeros, "all
// clear" pills and green dots — its healthiest possible appearance, describing a machine with
// no watchdogs at all. This is the one reading that distinguishes quiet from absent, and the
// page's own contract says absence must never be drawn as an all-clear.
//
// stability_watchdog.sh writes the heartbeat last in the chain, so a fresh value means a whole
// cycle finished rather than one that started and died halfway. Overdue is measured against the
// schedule in force (cfg.cycle_min, read from the cron), not a constant: one missed cycle is
// worth noticing, two is worth alarming about.
const cycleMin = cfg.cycle_min || 15;
const lastCyc = (node.states || {}).last_cycle || 0;
const cycAge = lastCyc ? Math.max(0, Math.floor(Date.now() / 1000) - lastCyc) : null;
const cycCol = cycAge == null ? '#555'
: cycAge > cycleMin * 60 * 3 ? '#ef5350'
: cycAge > cycleMin * 60 * 2 ? '#ffb74d' : '#4caf50';
const cycTxt = cycAge == null ? 'never' : _relTime(lastCyc);
const cycNote = cycAge == null ? ' — no cycle has completed'
: cycAge > cycleMin * 60 * 2 ? ' — overdue, expected every ' + cycleMin + 'm' : '';
const apiOnly = sys.api_only === true;
const daemonDot = sys.daemon_ok === null ? '#555' : sys.daemon_ok ? '#4caf50' : '#ef5350';
const daemonTxt = sys.daemon_ok === null ? '—' : sys.daemon_ok ? 'daemon ok' : 'daemon err';
@@ -439,6 +460,8 @@ function _systemCard(node, cfg) {
>${apiOnly ? 'API ONLY' : _levelLabel(level)}</span>
</div>
<div class="vv-wd-sec">System</div>
${!apiOnly ? _row('Last cycle', `<span style="color:${cycCol}">${vvEscHtml(cycTxt)}</span>`
+ `<span style="color:#3a3a3a;font-size:10px;">${vvEscHtml(cycNote)}</span>`) : ''}
${_row('RAM free', `<span style="color:${memCol}">${_fmtBytes(sys.mem_avail)}</span> / ${_fmtBytes(sys.mem_total)}`)}
${_bar(usedPct, memCol)}
<div style="display:flex;justify-content:space-between;margin-top:1px;font-size:10px;color:#333">