The Monitor fallback card described a state file without saying whether anything still writes it
This commit is contained in:
Gmer4Lfe
@@ -1,5 +1,6 @@
|
||||
<?php
|
||||
require_once __DIR__ . '/common.php';
|
||||
require_once __DIR__ . '/fallback.php'; // vv_fb_proc(), vv_fb_dryrun_state() — daemon liveness
|
||||
require_once __DIR__ . '/partnership.php'; // vv_pt_peer_match() — tailnet name vs conf hostname
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
@@ -126,6 +127,60 @@ function vv_partner_state(): array {
|
||||
];
|
||||
}
|
||||
|
||||
// Everything below the state file: whether anything is actually running, what it would do, and
|
||||
// when it last changed its mind. The card previously showed "✓ Nominal · monitoring · 30s" from
|
||||
// a state file alone — the same sentence whether a daemon was checking every 30 seconds or
|
||||
// nothing had run for five days.
|
||||
function vv_fb_card_extra(): array {
|
||||
$me = vv_detect_host();
|
||||
$vars = vv_conf_vars();
|
||||
|
||||
// Liveness and mode, read the same way the Fallback tab reads it.
|
||||
$proc = function_exists('vv_fb_proc')
|
||||
? vv_fb_proc('fallback')
|
||||
: ['running' => null, 'pid' => null, 'mode' => null, 'stale_lock' => false];
|
||||
|
||||
// A dry run keeps its own state copy, so the live file legitimately stops changing while a
|
||||
// preview runs. Reported so the card can say which file it is describing.
|
||||
$preview = ['state' => null, 'age' => null];
|
||||
if (($proc['mode'] ?? '') === 'dry-run' && !empty($proc['pid']) && function_exists('vv_fb_dryrun_state')) {
|
||||
$preview = vv_fb_dryrun_state((int)$proc['pid']);
|
||||
}
|
||||
|
||||
$sp = STATE_DIR . '/fallback_state.db';
|
||||
$lastChg = is_file($sp) ? time() - (int)@filemtime($sp) : null;
|
||||
|
||||
// What this host would take on if the partner went dark, and what the partner would take on
|
||||
// for us. Both are counts, not lists — the card is one column wide and the Fallback tab owns
|
||||
// the detail. Tier lists always live in the COVERED host's conf.
|
||||
$partner = null;
|
||||
foreach (vv_known_hosts() as $slot => $hostname) {
|
||||
if ($slot !== $me) { $partner = $slot; break; }
|
||||
}
|
||||
$weCover = $theyCover = null;
|
||||
if ($partner !== null && function_exists('vv_read_host_conf_raw')) {
|
||||
$pRaw = vv_read_host_conf_raw($partner);
|
||||
$myRaw = vv_read_host_conf_raw($me);
|
||||
$count = function (string $raw, string $id): int {
|
||||
$n = 0;
|
||||
for ($t = 1; $t <= 4; $t++) $n += count(vv_parse_bash_array($raw, "FALLBACK_{$id}_TIER{$t}"));
|
||||
return $n;
|
||||
};
|
||||
$weCover = $count($pRaw, strtoupper($partner)); // partner's tiers = what we start for them
|
||||
$theyCover = $count($myRaw, strtoupper($me)); // our tiers = what they start for us
|
||||
}
|
||||
|
||||
return [
|
||||
'daemon' => $proc,
|
||||
'preview' => $preview,
|
||||
'last_change' => $lastChg,
|
||||
'partner_id' => $partner !== null ? strtoupper($partner) : null,
|
||||
'we_cover' => $weCover,
|
||||
'they_cover' => $theyCover,
|
||||
'rsync_on_handback' => ($vars['FALLBACK_RSYNC_ENABLED'] ?? 'false') === 'true',
|
||||
];
|
||||
}
|
||||
|
||||
function vv_fallback_state(): array {
|
||||
$vars = vv_conf_vars();
|
||||
$enabled = ($vars['FALLBACK_ENABLED'] ?? 'false') === 'true';
|
||||
@@ -135,7 +190,7 @@ function vv_fallback_state(): array {
|
||||
|
||||
$stateFile = STATE_DIR . '/fallback_state.db';
|
||||
if (!file_exists($stateFile)) {
|
||||
return ['state' => 'UNKNOWN', 'enabled' => $enabled, 'check_interval' => $interval,
|
||||
return vv_fb_card_extra() + ['state' => 'UNKNOWN', 'enabled' => $enabled, 'check_interval' => $interval,
|
||||
'handback_strikes' => 0, 'handback_strikes_required' => $reqStrikes,
|
||||
'partnership_suspended' => false,
|
||||
'partner_lost_at' => 0, 'partnership_suspend_after' => $suspendAfter];
|
||||
@@ -145,7 +200,7 @@ function vv_fallback_state(): array {
|
||||
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
||||
$raw[trim($k)] = trim($v);
|
||||
}
|
||||
return [
|
||||
return vv_fb_card_extra() + [
|
||||
'state' => $raw['state'] ?? 'UNKNOWN',
|
||||
'failover_start' => $raw['failover_start'] ?? '0',
|
||||
'tier2_started' => $raw['tier2_started'] ?? 'false',
|
||||
|
||||
@@ -1212,6 +1212,49 @@ function vvPollMonitor(live) {
|
||||
}
|
||||
fbHtml += `<div style="font-size:11px;color:${sColor};font-weight:500;margin-bottom:6px;">${sLabel}${stateExtra}</div>`;
|
||||
|
||||
// ── Daemon liveness — shown in EVERY state ─────────────────────────────
|
||||
// The line above describes a file. This one describes whether anything is still writing
|
||||
// it. Without it the card read "✓ Nominal · monitoring · 30s" on a host where nothing had
|
||||
// run for five days: the state file said NORMAL, and nothing contradicted it.
|
||||
const dae = fb.daemon ?? {};
|
||||
let daeHtml;
|
||||
if (dae.stale_lock) {
|
||||
daeHtml = `<span style="color:#f44336;">✕ not running</span>
|
||||
<span style="color:#444;"> · stale lock${dae.pid ? ' (PID ' + dae.pid + ')' : ''}</span>`;
|
||||
} else if (dae.running === false) {
|
||||
daeHtml = `<span style="color:#f44336;">✕ not running</span>
|
||||
<span style="color:#444;"> · nothing is watching</span>`;
|
||||
} else if (dae.running && dae.mode === 'dry-run') {
|
||||
// A preview decides nothing and writes to its own state copy, so the live figures on
|
||||
// this card stop moving while it runs. Said plainly rather than left to infer.
|
||||
daeHtml = `<span style="color:#4a9eff;">◐ dry run</span>
|
||||
<span style="color:#444;"> · preview only, PID ${dae.pid}</span>`;
|
||||
} else if (dae.running) {
|
||||
daeHtml = `<span style="color:#4caf50;">◉ watching</span>
|
||||
<span style="color:#444;"> · every ${interval}s</span>`;
|
||||
} else {
|
||||
daeHtml = `<span style="color:#555;">— unknown</span>`;
|
||||
}
|
||||
fbHtml += `<div style="font-size:10px;margin-bottom:7px;">${daeHtml}</div>`;
|
||||
|
||||
// ── Mutual coverage ────────────────────────────────────────────────────
|
||||
// Both directions, because they are configured in different files and are routinely
|
||||
// asymmetric — this mesh currently has one side fully populated and the other empty,
|
||||
// which is invisible if the card only reports its own.
|
||||
const pid2 = fb.partner_id;
|
||||
if (pid2) {
|
||||
const we = fb.we_cover;
|
||||
const they = fb.they_cover;
|
||||
const num = (n, warn) => n === null || n === undefined
|
||||
? `<span style="color:#333;">—</span>`
|
||||
: `<span style="color:${n === 0 && warn ? '#ff9800' : '#888'};">${n}</span>`;
|
||||
fbHtml += `<div style="display:grid;grid-template-columns:1fr auto;gap:1px 8px;font-size:10px;margin-bottom:6px;">
|
||||
<span style="color:#444;">We cover ${vvEscHtml(pid2)}</span>${num(we, true)}
|
||||
<span style="color:#444;">${vvEscHtml(pid2)} covers us</span>${num(they, false)}
|
||||
</div>`;
|
||||
if (we === 0) fbHtml += `<div style="font-size:9px;color:#5a4020;margin-bottom:6px;">No tiers set for ${vvEscHtml(pid2)} — we would start nothing</div>`;
|
||||
}
|
||||
|
||||
// ── Handback strike progress ───────────────────────────────────────────
|
||||
if (state === 'FALLBACK' && strikes > 0) {
|
||||
const dots = Array.from({length: maxStrikes}, (_,i) =>
|
||||
@@ -1257,9 +1300,22 @@ function vvPollMonitor(live) {
|
||||
|
||||
// ── NORMAL quiet state ─────────────────────────────────────────────────
|
||||
if (state === 'NORMAL' && !fbActive.length) {
|
||||
let meta = `monitoring · ${interval}s`;
|
||||
if (ptRequired) meta += ' · partnership gated';
|
||||
fbHtml += `<div style="font-size:10px;color:#2a2a2a;">${meta}</div>`;
|
||||
// "monitoring · 30s" used to live here and was the card's only footer — a claim about a
|
||||
// running process made from a file. The daemon row above owns that claim now, so this
|
||||
// reports the things the state file genuinely knows.
|
||||
const bits = [];
|
||||
const lc = fb.last_change;
|
||||
if (lc !== null && lc !== undefined) {
|
||||
const a = lc < 60 ? lc + 's' : lc < 3600 ? Math.floor(lc/60) + 'm'
|
||||
: lc < 86400 ? Math.floor(lc/3600) + 'h' : Math.floor(lc/86400) + 'd';
|
||||
bits.push(`no change in ${a}`);
|
||||
}
|
||||
if (ptRequired) bits.push('partnership gated');
|
||||
// Only mentioned when OFF, and only here: handback writeback is the step that carries
|
||||
// work done during an outage back to the recovered host, and losing it silently is the
|
||||
// expensive half of a failover nobody notices until afterwards.
|
||||
if (fb.rsync_on_handback === false) bits.push('<span style="color:#5a4020;">no handback rsync</span>');
|
||||
if (bits.length) fbHtml += `<div style="font-size:9px;color:#2a2a2a;">${bits.join(' · ')}</div>`;
|
||||
}
|
||||
|
||||
// ── NO_INTERNET / DARK detail ──────────────────────────────────────────
|
||||
@@ -1276,6 +1332,12 @@ function vvPollMonitor(live) {
|
||||
if (fbCard) {
|
||||
fbCard.classList.remove('vv-accent-ok','vv-accent-warn','vv-accent-err');
|
||||
if (!enabled || ptSuspended) fbCard.classList.add('vv-accent-warn');
|
||||
// A green accent is a claim that this is covered. It is not, if nothing is running or
|
||||
// the only thing running is a preview that would take no action.
|
||||
else if (fb.daemon && (fb.daemon.running === false || fb.daemon.stale_lock))
|
||||
fbCard.classList.add('vv-accent-err');
|
||||
else if (fb.daemon && fb.daemon.mode === 'dry-run')
|
||||
fbCard.classList.add('vv-accent-warn');
|
||||
else if (state === 'NORMAL') fbCard.classList.add('vv-accent-ok');
|
||||
else if (state === 'FALLBACK') fbCard.classList.add('vv-accent-err');
|
||||
else if (state !== 'UNKNOWN') fbCard.classList.add('vv-accent-warn');
|
||||
|
||||
Reference in New Issue
Block a user