diff --git a/Plugin/unraid/Tools/api_cache_writer.php b/Plugin/unraid/Tools/api_cache_writer.php index 10339d3..c79effe 100644 --- a/Plugin/unraid/Tools/api_cache_writer.php +++ b/Plugin/unraid/Tools/api_cache_writer.php @@ -147,5 +147,12 @@ vv_cache_write('monitor', $monitor); $arrs = vv_arrs_all(); vv_cache_write('arrs', $arrs); +// ── Watchdog payload ────────────────────────────────────────────────────────── +// Here for the same reason as the other two: vv_wd_all() SSHes to every partner, which cost 8.3s +// on this host with HOST2 down, and the Watchdog tab polls every 30 seconds. Collected off the +// request path it is paid once a minute by cron instead of by whoever has the tab open. +require_once $_base . '/include/watchdog.php'; +vv_cache_write('watchdog', vv_wd_all()); + $elapsed = round((microtime(true) - $t) * 1000); -echo "Cache written in {$elapsed}ms — monitor + arrs\n"; +echo "Cache written in {$elapsed}ms — monitor + arrs + watchdog\n"; diff --git a/Plugin/unraid/api/watchdog.php b/Plugin/unraid/api/watchdog.php index b8bf731..042833a 100644 --- a/Plugin/unraid/api/watchdog.php +++ b/Plugin/unraid/api/watchdog.php @@ -33,15 +33,41 @@ // Remote collection degrades per node — one unreachable partner drops that node's card and // leaves the local host and every other partner intact. // +// Served from cache, collected only on a miss. +// vv_wd_all() SSHes to every configured partner, so its cost is set by the slowest node +// rather than by how much data there is. Measured at 8.3s on this host with HOST2 down — +// paid by every visitor, every 30 seconds, because the tab polls. The cache check happens +// before the heavy include, so a hit costs one file read and no SSH at all. +// +// The window is deliberately wider than the poll. Watchdog state only changes when the +// orchestrator runs, which is every 15 minutes; polling it every 30 seconds was never +// reading anything new, it was just re-paying for the same answer. +// // REQUEST -// GET, no parameters +// GET served from the 300s cache when one is present +// GET ?live bypass the cache and collect everything fresh // // RESPONSE // vv_wd_all() verbatim — per-node watchdog state plus the resolved threshold set // // DEPENDS ON -// include/watchdog.php vv_wd_all() +// include/config.php vv_cache_read() +// include/watchdog.php vv_wd_all() — required only on a miss +// Tools/api_cache_writer.php writes the cache this endpoint normally serves // ═══════════════════════════════════════════════════════════════════════════════════════════════ header('Content-Type: application/json'); +require_once dirname(__DIR__) . '/include/config.php'; + +if (!isset($_GET['live'])) { + $_vv_cached = vv_cache_read('watchdog', 300); + if ($_vv_cached !== null) { echo json_encode($_vv_cached); exit; } + unset($_vv_cached); +} + require_once dirname(__DIR__) . '/include/watchdog.php'; -echo json_encode(vv_wd_all()); +$_vv_wd = vv_wd_all(); +// Written on the miss as well as by the background writer. Without this the first visitor after +// a restart pays the full collection and so does the next one, until the writer's next minute +// happens to land. +vv_cache_write('watchdog', $_vv_wd); +echo json_encode($_vv_wd); diff --git a/Plugin/unraid/include/config.php b/Plugin/unraid/include/config.php index b343bcb..09bdc86 100644 --- a/Plugin/unraid/include/config.php +++ b/Plugin/unraid/include/config.php @@ -648,9 +648,25 @@ function vv_format_uptime(int $seconds): string { // Parse a scalar value from raw conf text. Matches KEY="value" or KEY=value. // Identical logic was previously duplicated as vv_arr_scalar / vv_wd_scalar / // vv_fb_scalar / vv_media_conf_scalar — all reduce to this one regex. +// +// Trailing comments are stripped, which the single-regex version did not do. It captured to the +// end of the line, so `FALLBACK_ENABLED=true # HOST2 back online` parsed as the whole string +// `true # HOST2 back online`. Numeric readings survived that — (int) and (float) stop at the +// first non-digit — which is why it went unnoticed: every threshold was right and every boolean +// was wrong. `=== 'true'` was false for any commented var, and worse, `!== 'false'` was TRUE for +// one, so a commented-out-to-off switch read as on. Roughly half of master.conf's toggles carry +// an inline comment. +// +// The quoted forms are extracted before that, because inside quotes a # is data, not a comment — +// a password or a colour would otherwise be truncated at the first hash. Unquoted, the comment +// must be introduced by whitespace, matching bash: FOO=#fff and FOO=bar#baz both assign literally, +// since # only opens a comment at the start of a word. function vv_parse_conf_scalar(string $raw, string $key): string { - return preg_match('/^\s*' . preg_quote($key, '/') . '\s*=\s*"?([^"\n]*)"?/m', $raw, $m) - ? trim($m[1]) : ''; + if (!preg_match('/^\s*' . preg_quote($key, '/') . '\s*=\s*(.*)$/m', $raw, $m)) return ''; + $v = ltrim($m[1]); + if (preg_match('/^"([^"\n]*)"/', $v, $q)) return $q[1]; + if (preg_match("/^'([^'\n]*)'/", $v, $q)) return $q[1]; + return trim(preg_replace('/\s+#.*$/', '', $v)); } // Parse a key=value state file (e.g. fallback_state.db, partnership_state.db). diff --git a/Plugin/unraid/include/watchdog.php b/Plugin/unraid/include/watchdog.php index ad7cef2..2c36f99 100644 --- a/Plugin/unraid/include/watchdog.php +++ b/Plugin/unraid/include/watchdog.php @@ -56,6 +56,18 @@ require_once __DIR__ . '/partnership.php'; // Watchdog tab data helpers +// Rows in system_watchdog_state.db that are bookkeeping, not strikes. The file is a flat key/value +// store shared by the counters and the watchdog's own liveness record, so the only thing separating +// the two is knowing which is which — there is no marker in the format. +// +// watchdog_cycle is a unix timestamp, written every cycle by stability_watchdog.sh:883 for +// docker_watchdog's stale-state guard. Published as a strike it reads as 1.7 billion of them. +// +// Add to this list, never to a shape test: a strike count and a timestamp are both positive +// integers, and a threshold like "bigger than a billion means it is a clock" is a rule that works +// until it doesn't and fails silently in the direction of hiding a real strike. +const VV_WD_SYS_BOOKKEEPING = ['watchdog_cycle']; + // ── Conf array parser (bash arrays) ────────────────────────────────────────── function vv_wd_bash_array(string $raw, string $varname): array { @@ -213,11 +225,19 @@ function vv_wd_local_states(string $restartLogPath): array { $strikes[$k] = (int)$v; } - // Stability strikes: everything in sys state that isn't a flag key + // Stability strikes: everything in sys state that is actually a strike. + // + // The old guard was `!str_contains($k, '=')`, written on the belief that the bookkeeping rows + // in system_watchdog_state.db keep their `=` inside the key because the file mixes separators + // — `ram:0` and `sshd:0` beside `kernel_oops_count=0` and `watchdog_cycle=...`. They do not: + // vv_wd_parse_kv() splits on both, so the guard never matched anything and every bookkeeping + // row was published as a strike. watchdog_cycle is a unix timestamp written once per cycle by + // stability_watchdog.sh:883 as a liveness heartbeat, so the Stability card was drawing + // "watchdog cycle — 1786716931" next to a limit of 2. $sysStrikes = []; foreach ($sys as $k => $v) { - if (!str_contains($k, '=') && (int)$v > 0) - $sysStrikes[$k] = (int)$v; + if (in_array($k, VV_WD_SYS_BOOKKEEPING, true)) continue; + if ((int)$v > 0) $sysStrikes[$k] = (int)$v; } // Growth baseline info (container count + age in seconds) @@ -322,9 +342,12 @@ function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath): if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0) $strikes[$k] = (int)$v; } + // Same exclusion as the local path above — a partner's heartbeat is no more a strike than + // this host's. The two builders are maintained together. $sysStrikes = []; foreach ($sys as $k => $v) { - if (!str_contains($k, '=') && (int)$v > 0) $sysStrikes[$k] = (int)$v; + if (in_array($k, VV_WD_SYS_BOOKKEEPING, true)) continue; + if ((int)$v > 0) $sysStrikes[$k] = (int)$v; } $baselineCount = (int)($hdr['BASELINECOUNT'] ?? 0); @@ -420,6 +443,12 @@ function vv_wd_all(): array { 'net_wd_enabled' => vv_wd_scalar($masterRaw, 'NETWORK_WATCHDOG_ENABLED') !== 'false', 'npm_strike_lim' => (int)(vv_wd_scalar($masterRaw, 'NETWORK_WATCHDOG_NPM_STRIKE_LIMIT') ?: 2), 'ts_check' => vv_wd_scalar($masterRaw, 'NETWORK_WATCHDOG_CHECK_TAILSCALE') !== 'false', + // Conf cache watchdog. It has no state of its own to report because it writes none until + // it runs, and it does not run: conf_cache_watchdog.sh:145-146 exits on either of these + // being off. Both gates ship so the page can say "off, and here is which switch", rather + // than omit a member of SYSTEM_WATCHDOG_SCRIPTS entirely and let its absence read as fine. + 'fallback_on' => vv_wd_scalar($masterRaw, 'FALLBACK_ENABLED') === 'true', + 'conf_sync_on' => vv_wd_scalar($masterRaw, 'CONF_SYNC_ENABLED') !== 'false', ]; // SSH key from current host conf diff --git a/Plugin/unraid/pages/watchdog.php b/Plugin/unraid/pages/watchdog.php index 8aef555..28e1194 100644 --- a/Plugin/unraid/pages/watchdog.php +++ b/Plugin/unraid/pages/watchdog.php @@ -270,6 +270,17 @@ function _systemCard(node, cfg) { : sys.load1 > sys.cores * cfg.rw_load_soft ? '#ffb74d' : '#4caf50'; + // The fourth member of SYSTEM_WATCHDOG_SCRIPTS, and the only one with nothing to show: it keeps + // the persistent backup of the partner's conf current, and writes no state until it does. Drawn + // as its gate rather than left off the card, because a chain member that is simply missing from + // the page reads as one that is fine — the same reason an unreachable node here says so instead + // of rendering healthy. Names the switch, so "why is it off" does not need a hunt through conf. + const confCacheHtml = !cfg.fallback_on + ? 'off — FALLBACK_ENABLED' + : !cfg.conf_sync_on + ? 'off — CONF_SYNC_ENABLED' + : 'active'; + const apiOnly = sys.api_only === true; const daemonDot = sys.daemon_ok === null ? '#555' : sys.daemon_ok ? '#4caf50' : '#ef5350'; const daemonTxt = sys.daemon_ok === null ? '—' : sys.daemon_ok ? 'daemon ok' : 'daemon err'; @@ -302,6 +313,7 @@ function _systemCard(node, cfg) {
` : ''} ${_row('Uptime', sys.uptime ? _dur(sys.uptime) : '—')} ${_row('Docker', `${daemonTxt}`)} + ${_row('Conf cache', confCacheHtml)} ${!apiOnly ? (sys.oom_count > 0 ? _row('OOM kills', `${sys.oom_count}`) : _row('OOM kills', '0')) : ''} ${apiOnly ? `