diff --git a/Plugin/unraid/include/watchdog.php b/Plugin/unraid/include/watchdog.php index ff43af1..da3e710 100644 --- a/Plugin/unraid/include/watchdog.php +++ b/Plugin/unraid/include/watchdog.php @@ -84,6 +84,25 @@ function vv_wd_installed_containers(): array { return array_values(array_filter(array_map('trim', $out), fn($n) => $n !== '')); } +// How many minutes are supposed to pass between watchdog cycles. +// +// Parsed from varaverk.cron, which the scheduler regenerates at array start and is the only place +// the answer exists — the interval is a schedule, not a conf key. Reading it rather than assuming +// 15 means that changing the schedule also moves the point at which the page calls a cycle +// overdue, instead of leaving a constant here to drift out of agreement with reality. +// +// Falls back to 15 when the cron is missing or the entry is written in a form this does not +// recognise. A wrong-but-sane interval produces a slightly early or late warning; refusing to +// answer would remove the liveness check altogether, which is the failure being fixed. +function vv_wd_cycle_interval_min(): int { + $cron = @file_get_contents('/boot/config/plugins/varaverk/varaverk.cron') ?: ''; + foreach (explode("\n", $cron) as $line) { + if (!str_contains($line, 'watchdog_orchestrator')) continue; + if (preg_match('#^\s*\*/(\d+)\s#', $line, $m)) return max(1, (int)$m[1]); + } + return 15; +} + // ── Conf array parser (bash arrays) ────────────────────────────────────────── function vv_wd_bash_array(string $raw, string $varname): array { @@ -272,6 +291,9 @@ function vv_wd_local_states(string $restartLogPath): array { 'ctr_strikes' => $strikes, 'skiplist' => vv_wd_parse_skiplist($skipRaw), 'sys_strikes' => $sysStrikes, + // The heartbeat, not a strike — see VV_WD_SYS_BOOKKEEPING. stability_watchdog.sh writes it + // last in the chain, so a fresh value means a whole cycle completed rather than started. + 'last_cycle' => (int)($sys['watchdog_cycle'] ?? 0), 'reboots' => vv_wd_parse_reboot_log($rebootRaw), 'restarts' => vv_wd_parse_restart_log($restartRaw), 'storage_wd' => vv_wd_parse_storage_state($storRaw) + [ @@ -391,6 +413,7 @@ function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath): 'ctr_strikes' => $strikes, 'skiplist' => vv_wd_parse_skiplist($skipRaw), 'sys_strikes' => $sysStrikes, + 'last_cycle' => (int)($sys['watchdog_cycle'] ?? 0), 'reboots' => vv_wd_parse_reboot_log($rebootRaw), 'restarts' => vv_wd_parse_restart_log($restartRaw), 'storage_wd' => vv_wd_parse_storage_state($storRaw) + [ @@ -465,6 +488,10 @@ function vv_wd_all(): array { // than omit a member of SYSTEM_WATCHDOG_SCRIPTS entirely and let its absence read as fine. 'fallback_on' => vv_wd_scalar($masterRaw, 'FALLBACK_ENABLED') === 'true', 'conf_sync_on' => vv_wd_scalar($masterRaw, 'CONF_SYNC_ENABLED') !== 'false', + // How often a cycle is supposed to happen, so "overdue" is measured against the schedule + // in force rather than a number written here. Read from the cron because that is where + // the answer lives — there is no conf key for it. + 'cycle_min' => vv_wd_cycle_interval_min(), ]; // SSH key from current host conf diff --git a/Plugin/unraid/pages/watchdog.php b/Plugin/unraid/pages/watchdog.php index bcded00..6711a9b 100644 --- a/Plugin/unraid/pages/watchdog.php +++ b/Plugin/unraid/pages/watchdog.php @@ -419,6 +419,27 @@ function _systemCard(node, cfg) { ? 'off — CONF_SYNC_ENABLED' : 'active'; + // ── Is anything still running? ────────────────────────────────────────────── + // Every other number on this card is only meaningful if a cycle actually happened. If the + // orchestrator stopped, the strike maps stop changing and the whole page renders zeros, "all + // clear" pills and green dots — its healthiest possible appearance, describing a machine with + // no watchdogs at all. This is the one reading that distinguishes quiet from absent, and the + // page's own contract says absence must never be drawn as an all-clear. + // + // stability_watchdog.sh writes the heartbeat last in the chain, so a fresh value means a whole + // cycle finished rather than one that started and died halfway. Overdue is measured against the + // schedule in force (cfg.cycle_min, read from the cron), not a constant: one missed cycle is + // worth noticing, two is worth alarming about. + const cycleMin = cfg.cycle_min || 15; + const lastCyc = (node.states || {}).last_cycle || 0; + const cycAge = lastCyc ? Math.max(0, Math.floor(Date.now() / 1000) - lastCyc) : null; + const cycCol = cycAge == null ? '#555' + : cycAge > cycleMin * 60 * 3 ? '#ef5350' + : cycAge > cycleMin * 60 * 2 ? '#ffb74d' : '#4caf50'; + const cycTxt = cycAge == null ? 'never' : _relTime(lastCyc); + const cycNote = cycAge == null ? ' — no cycle has completed' + : cycAge > cycleMin * 60 * 2 ? ' — overdue, expected every ' + cycleMin + 'm' : ''; + const apiOnly = sys.api_only === true; const daemonDot = sys.daemon_ok === null ? '#555' : sys.daemon_ok ? '#4caf50' : '#ef5350'; const daemonTxt = sys.daemon_ok === null ? '—' : sys.daemon_ok ? 'daemon ok' : 'daemon err'; @@ -439,6 +460,8 @@ function _systemCard(node, cfg) { >${apiOnly ? 'API ONLY' : _levelLabel(level)}