From 0f228020efd9086c4f80e0ef2cc71d19daafa1a3 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sat, 22 Aug 2026 00:52:38 -0400 Subject: [PATCH] State-file age is time since the last transition, not staleness, and a preview's state is the one being decided --- Plugin/unraid/include/fallback.php | 35 +++++++++++++++ Plugin/unraid/pages/fallback.php | 68 +++++++++++++++++++----------- 2 files changed, 79 insertions(+), 24 deletions(-) diff --git a/Plugin/unraid/include/fallback.php b/Plugin/unraid/include/fallback.php index 5e293f2..6493d47 100644 --- a/Plugin/unraid/include/fallback.php +++ b/Plugin/unraid/include/fallback.php @@ -172,6 +172,30 @@ function vv_fb_remote_proc(string $ip, string $sshKey): array { } return $res; } + +// A running dry run keeps its own state file — VV_CACHE_ROOT/fallback_state.dryrun. — and +// refreshes it every check interval, exactly as the live daemon would. Reading the LIVE file +// while a preview is running is how a healthy dry run came to render as UNKNOWN on a host that +// has simply never run fallback for real: the answer existed, in a file next to the one being +// read. The preview is shown as the preview, never merged into the live state. +function vv_fb_dryrun_state(int $pid): array { + $p = (VV_CACHE_ROOT ?: '/tmp/varaverk') . '/fallback_state.dryrun.' . $pid; + if (!is_file($p)) return ['state' => null, 'age' => null]; + $s = vv_fb_parse_state((string)@file_get_contents($p)); + $s['age'] = time() - (int)@filemtime($p); + return $s; +} + +function vv_fb_remote_dryrun_state(string $ip, string $sshKey, int $pid): array { + $f = '/tmp/varaverk/fallback_state.dryrun.' . $pid; + $out = vv_pt_ssh($ip, $sshKey, "[ -f '$f' ] && { echo \"__age=\$(( \$(date +%s) - \$(stat -c %Y '$f') ))\"; cat '$f'; }"); + if (trim((string)$out) === '') return ['state' => null, 'age' => null]; + $age = null; + if (preg_match('/^__age=(\d+)/m', (string)$out, $m)) $age = (int)$m[1]; + $s = vv_fb_parse_state((string)$out); + $s['age'] = $age; + return $s; +} // ── Covers — what a node runs for the other when it's down ─────────────────── function vv_fb_covers(string $covering, string $remote, string $coveringRaw, string $remoteRaw): array { @@ -279,6 +303,16 @@ function vv_fb_all(): array { $procTest = ['running' => null, 'pid' => null, 'mode' => null, 'stale_lock' => false]; } + // When a preview is running, read what IT is deciding — kept beside the live file and + // refreshed on the same interval. Reported separately so the live state is never + // overwritten by a preview's opinion. + $preview = ['state' => null, 'age' => null]; + if (($proc['mode'] ?? '') === 'dry-run' && !empty($proc['pid'])) { + $preview = $isMe + ? vv_fb_dryrun_state((int)$proc['pid']) + : vv_fb_remote_dryrun_state($ip, $mySshKey, (int)$proc['pid']); + } + // How fresh the state actually is. The daemon rewrites its file every check interval, so // an age far past that interval means it is wedged even while the process still exists. $stateAge = null; @@ -304,6 +338,7 @@ function vv_fb_all(): array { 'ts_ip' => $ip, 'state' => $state, 'state_age' => $stateAge, + 'preview' => $preview, 'running' => $running, 'running_count' => count($running), 'proc' => $proc, diff --git a/Plugin/unraid/pages/fallback.php b/Plugin/unraid/pages/fallback.php index acb8c63..36e22f7 100644 --- a/Plugin/unraid/pages/fallback.php +++ b/Plugin/unraid/pages/fallback.php @@ -24,9 +24,15 @@ // A missing state file renders as unknown, never as NORMAL — claiming healthy for a // fallback process that is not running would be the worst possible error on this page. // -// State age is shown next to state, always. Every value in the state file was written by a -// daemon that may not be running: this host displayed NORMAL from a file five days stale -// with no process alive, and nothing on the page said so. +// Liveness is the PROCESS, never the state file's age. fallback.sh writes that file only on a +// transition — the steady NORMAL path writes nothing — so an untouched file means nothing has +// changed, which is the healthy case. An earlier version of this card read that age as +// staleness and coloured a perfectly good five-day-quiet host red. The row is "Last change" +// and carries no verdict; "is anything watching" is answered by _daemonRow reading /proc. +// +// A running preview's state is shown in place of the live value, marked "preview". A dry run +// redirects its writes to its own copy, so reading the live file while one runs reported a +// healthy preview as UNKNOWN on a host that has never run fallback for real. // // Stopping the test is a different button from stopping the daemon, deliberately. The test // holds an iptables rule that only its own EXIT trap removes, so the two cannot share a @@ -423,26 +429,24 @@ function _daemonRow(p, checkInterval) { return ['good', mode + ' · PID ' + p.pid, 'up']; } -// State freshness. The daemon rewrites this file every check interval, so anything past a few -// intervals is a wedged process or a dead one — and the state value it left behind is a claim -// about the past, not the present. -function _freshness(node, checkInterval) { - const a = node.state_age; - // A dry run redirects its state writes to a per-PID copy so the live file is never touched. - // That is the point of it — but it means the live file goes on ageing while a preview runs, - // and "daemon: dry run" beside "state age: 5d — stale" reads as a fault when it is the - // designed behaviour. Say which it is rather than leaving the operator to chase it. +// Time since the last STATE CHANGE — not staleness, and not a heartbeat. +// +// fallback.sh writes this file only on a transition: the steady NORMAL path logs and writes +// nothing. So a file untouched for five days means "nothing has changed in five days", which on +// this page is the healthy case, and an earlier version of this card labelled exactly that as +// "stale" and coloured it red. Liveness is the PROCESS, which _daemonRow reads directly and is +// the only thing that can answer "is anything watching right now". +function _lastChange(node) { const dry = node.proc && node.proc.running && node.proc.mode === 'dry-run'; + const a = dry ? (node.preview?.age ?? null) : node.state_age; if (a === null || a === undefined) { - if (dry) return ['dim', 'none — dry run does not write live state']; + if (dry) return ['dim', '—']; return node.is_me - ? ['bad', 'no state file — fallback has never run here'] + ? ['dim', 'never — no state file yet'] : ['dim', 'not readable from here']; } - const limit = Math.max(120, (checkInterval || 30) * 4); - if (a > limit && dry) return ['dim', _age(a) + ' — not refreshed: dry run writes elsewhere']; - return [a > limit ? 'bad' : 'good', _age(a) + (a > limit ? ' — stale' : '')]; + return ['dim', _age(a)]; } function _leg(ok, label) { @@ -459,7 +463,11 @@ function _nodeCard(node, data) { const slot = node.slot; const [dCls, dTxt, dKind] = _daemonRow(node.proc, data.check_interval); - const [fCls, fTxt] = _freshness(node, data.check_interval); + const [fCls, fTxt] = _lastChange(node); + // A preview running beside an absent or unchanging live state is what is actually being + // decided right now, so it is what the badge shows — labelled, never merged into live state. + const dryRun = node.proc && node.proc.running && node.proc.mode === 'dry-run'; + const shown = (dryRun && node.preview && node.preview.state) ? node.preview.state : state; const test = node.proc_test || {}; const reach = node.reach || {}; @@ -503,8 +511,8 @@ function _nodeCard(node, data) { ${vvEscHtml(node.hostname)} ${node.is_me ? 'US' : ''} - ${(node.is_me && fCls === 'good') ? _ptStatus(st) : ''} - ${_stateBadge(state)} + ${(node.is_me && node.proc && node.proc.running && node.proc.mode === 'live') ? _ptStatus(st) : ''} + ${_stateBadge(shown)}${dryRun ? 'preview' : ''}
@@ -516,7 +524,7 @@ function _nodeCard(node, data) {
Daemon ${dTxt} - State age ${fTxt} + Last change${fTxt} Covers ${covTarget} Running ${node.running_count ?? 0} containers ${state === 'FALLBACK' @@ -552,9 +560,21 @@ function _verdict(data) { if (inFb.length) return void (el.innerHTML = `${inFb.map(n=>n.id).join(', ')} in FALLBACK — covering for a partner right now.`); - let s = `Armed · ${live.length}/${nodes.length} node${nodes.length!==1?'s':''} running the daemon`; - if (dry.length) s += ` · ${dry.length} in dry run`; - if (dead.length) s += ` · ${dead.map(n=>n.id).join(', ')} not running — nothing would detect an outage there`; + // Count every running daemon, then say what KIND. Splitting live from dry-run and reporting + // only the live count read as "0/2 nodes running the daemon · 2 in dry run" — two clauses + // contradicting each other about the same two processes. + const up = live.length + dry.length; + let s = up === nodes.length + ? `Armed · all ${nodes.length} node${nodes.length!==1?'s':''} running` + : `Armed · ${up}/${nodes.length} node${nodes.length!==1?'s':''} running`; + + if (dry.length && !live.length) + s += ` — all previewing (dry run), nothing would actually fail over`; + else if (dry.length) + s += ` · ${dry.map(n=>n.id).join(', ')} previewing (dry run)`; + + if (dead.length) + s += ` · ${dead.map(n=>n.id).join(', ')} not running — nothing would detect an outage there`; el.innerHTML = s; } function _setToggles(data) {