Make fallback coverage something that can actually happen, and say so on the page
fallback.sh starts covered containers with docker start and never creates them, so a coverage list the partner has never been sent is a promise nothing can keep — all twelve were missing. Adds the push and remove paths, a readiness card that checks rather than infers, and the fallback state the assistant needs to answer for it.
This commit is contained in:
Gmer4Lfe
@@ -422,6 +422,17 @@ if ($can('system_state')) {
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback is dormant until it isn't, so "configured" and "would work" are unrelated — this is the
|
||||
// only block that reports the second. Cheap: local conf and state file plus one cached presence
|
||||
// read, never a network round trip.
|
||||
if ($can('fallback_state')) {
|
||||
$fb = vv_ai_fallback_state();
|
||||
if ($fb !== '') {
|
||||
$attached['fallback'] = 'readiness';
|
||||
$diagBlock .= $fb;
|
||||
}
|
||||
}
|
||||
|
||||
// The troubleshooting profile gets the actual tail of the one log the operator is looking at,
|
||||
// warnings and ordinary lines alike. The fleet-wide WARN/ERROR sweep above cannot answer "why
|
||||
// did this one stop" — the last line a script printed before dying is usually not labelled.
|
||||
|
||||
@@ -59,6 +59,7 @@
|
||||
// include/config.php vv_detect_host(), vv_read_conf_raw(), vv_push_master_conf()
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/fallback.php'; // vv_fb_proc()
|
||||
require_once dirname(__DIR__) . '/include/confform.php';
|
||||
require_once dirname(__DIR__) . '/include/common.php';
|
||||
|
||||
@@ -143,6 +144,191 @@ if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
||||
}
|
||||
|
||||
// ── Write ────────────────────────────────────────────────────────────────────────────────────
|
||||
// ── Readiness: would a failover actually work right now ─────────────────────────────────────
|
||||
// Every row is a deterministic check with a stated basis. The assistant on this page EXPLAINS
|
||||
// these rows; it never produces them. A model must not be the thing that says failover is ready —
|
||||
// that is precisely the class of answer this codebase keeps finding to be confidently wrong, and
|
||||
// on 2026-08-23 the coverage card itself was the confidently wrong surface: 12 containers listed,
|
||||
// none of them present on the partner.
|
||||
//
|
||||
// Rows are ordered by what breaks first, not by severity, so reading top to bottom follows the
|
||||
// order a real outage would hit them.
|
||||
if (($_POST['action'] ?? '') === 'readiness') {
|
||||
$rows = [];
|
||||
$add = function (string $id, string $label, string $verdict, string $detail, string $ask = '')
|
||||
use (&$rows) {
|
||||
// verdict: ok | warn | fail | unknown — unknown is never dressed up as ok
|
||||
$rows[] = ['id' => $id, 'label' => $label, 'verdict' => $verdict,
|
||||
'detail' => $detail, 'ask' => $ask];
|
||||
};
|
||||
|
||||
$me = vv_detect_host();
|
||||
$ME = strtoupper($me);
|
||||
$conf = vv_read_conf_raw('master.conf');
|
||||
$hc = vv_read_conf_raw($me . '.conf');
|
||||
|
||||
// 1. is fallback even armed
|
||||
$fbEnabled = preg_match('/^\s*FALLBACK_ENABLED\s*=\s*"?(\w+)/m', $conf, $m) ? $m[1] : 'unset';
|
||||
$add('enabled', 'Fallback armed',
|
||||
$fbEnabled === 'true' ? 'ok' : 'fail',
|
||||
'FALLBACK_ENABLED=' . $fbEnabled,
|
||||
'FALLBACK_ENABLED is ' . $fbEnabled . ' — what does that mean for a real outage?');
|
||||
|
||||
// 2. current state — anything but NORMAL means it is already doing something
|
||||
$stateFile = STATE_DIR . '/fallback_state.db';
|
||||
$state = 'unknown';
|
||||
if (is_readable($stateFile) && preg_match('/^state=(\S+)/m', (string)@file_get_contents($stateFile), $m)) {
|
||||
$state = $m[1];
|
||||
}
|
||||
// Same rule the node cards use: no state file plus a live daemon means the node has simply
|
||||
// never transitioned, which is health, not ignorance. Reading the file alone gives a healthy
|
||||
// node the same verdict as one whose daemon is dead.
|
||||
$daemon = function_exists('vv_fb_proc') ? (vv_fb_proc('fallback')['running'] ?? false) : false;
|
||||
$inferred = false;
|
||||
if ($state === 'unknown' && $daemon) { $state = 'NORMAL'; $inferred = true; }
|
||||
|
||||
$add('state', 'State is NORMAL',
|
||||
$state === 'NORMAL' ? 'ok' : ($state === 'unknown' ? 'unknown' : 'warn'),
|
||||
'state=' . $state . ($inferred ? ' (from the live daemon — never transitioned)' : ''),
|
||||
'Fallback state is ' . $state . '. What does that mean and what should I check?');
|
||||
|
||||
// 3. coverage configured at all
|
||||
$covered = [];
|
||||
for ($t = 1; $t <= 4; $t++) {
|
||||
foreach (vv_parse_conf_list($hc, "FALLBACK_{$ME}_TIER{$t}") as $c) $covered[] = $c;
|
||||
}
|
||||
$add('coverage', 'Containers are covered',
|
||||
$covered ? 'ok' : 'fail',
|
||||
$covered ? count($covered) . ' container(s) across the tiers' : 'no containers in any tier',
|
||||
$covered
|
||||
? 'Walk me through what happens if this host goes dark right now, tier by tier, with the delays.'
|
||||
: 'Nothing is listed in my fallback tiers — what would happen if this host went dark?');
|
||||
|
||||
// 4. THE one that was silently false — does the partner actually hold them
|
||||
$cache = (defined('VV_CACHE_ROOT') ? VV_CACHE_ROOT : '/tmp/varaverk') . '/api/fallback_presence.json';
|
||||
if (!is_readable($cache)) {
|
||||
$add('present', 'Partner has the containers', 'unknown',
|
||||
'never checked — run the presence check',
|
||||
'How do I find out whether the partner actually has my covered containers?');
|
||||
} else {
|
||||
$j = json_decode((string)@file_get_contents($cache), true);
|
||||
$miss = (array)($j['missing'] ?? []);
|
||||
$age = time() - (int)@filemtime($cache);
|
||||
$when = $age < 3600 ? round($age / 60) . 'm ago' : round($age / 3600) . 'h ago';
|
||||
$add('present', 'Partner has the containers',
|
||||
$miss ? 'fail' : 'ok',
|
||||
$miss ? count($miss) . ' of ' . count($covered) . ' missing (' . $when . '): '
|
||||
. implode(', ', array_slice($miss, 0, 4)) . (count($miss) > 4 ? '…' : '')
|
||||
: 'all ' . count($covered) . ' present (' . $when . ')',
|
||||
$miss ? 'The partner is missing ' . implode(', ', array_slice($miss, 0, 6))
|
||||
. '. What happens during a failover, and how do I fix it?' : '');
|
||||
}
|
||||
|
||||
// 5. host-specific networks that cannot be recreated on the partner
|
||||
$wg = [];
|
||||
foreach ($covered as $c) {
|
||||
foreach (glob('/boot/config/plugins/dockerMan/templates-user/*.xml') as $x) {
|
||||
$t = @file_get_contents($x);
|
||||
if ($t === false || strpos($t, "<Name>$c</Name>") === false) continue;
|
||||
if (preg_match('~<Network>(wg\d+)</Network>~', $t, $m)) $wg[] = "$c ({$m[1]})";
|
||||
break;
|
||||
}
|
||||
}
|
||||
if ($wg) {
|
||||
$add('wgnet', 'No tunnel-bound networks', 'warn',
|
||||
implode(', ', $wg),
|
||||
'Some covered containers use a WireGuard-backed network. Why can that not move to the partner?');
|
||||
}
|
||||
|
||||
// 6. handback writeback — invisible until the day it matters
|
||||
$wb = preg_match('/^\s*FALLBACK_RSYNC_ENABLED\s*=\s*"?(\w+)/m', $conf, $m) ? $m[1] : 'unset';
|
||||
$add('writeback', 'Handback writeback', $wb === 'true' ? 'ok' : 'warn',
|
||||
'FALLBACK_RSYNC_ENABLED=' . $wb,
|
||||
'FALLBACK_RSYNC_ENABLED is ' . $wb . ' — what do I lose on handback?');
|
||||
|
||||
// Overall verdict is the worst row, never an average. One failed check is a failed failover.
|
||||
$order = ['ok' => 0, 'warn' => 1, 'unknown' => 2, 'fail' => 3];
|
||||
$worst = 'ok';
|
||||
foreach ($rows as $r) if ($order[$r['verdict']] > $order[$worst]) $worst = $r['verdict'];
|
||||
|
||||
echo json_encode(['ok' => true, 'verdict' => $worst, 'rows' => $rows,
|
||||
'summary' => $worst === 'ok'
|
||||
? 'Every check passed'
|
||||
: ($worst === 'fail' ? 'A failover would NOT work as configured'
|
||||
: 'Failover is configured but something needs a look')]);
|
||||
exit;
|
||||
}
|
||||
// ── Push / remove / status: what the PARTNER actually holds ──────────────────────────────────
|
||||
// Coverage names a container; fallback.sh starts it with `docker start`, which fails unless the
|
||||
// partner already has it built. Measured 2026-08-23: 12 of 12 covered containers were absent from
|
||||
// the partner, so every tier would have failed on the first real outage. These three actions are
|
||||
// how the card closes and inspects that gap.
|
||||
//
|
||||
// Deliberately NOT folded into `cover`. Saving a tier list is a cheap, reversible config write;
|
||||
// deploying a dozen containers onto another machine is neither, and a stray click should not be
|
||||
// able to do it.
|
||||
$_covAction = $_POST['action'] ?? '';
|
||||
if (in_array($_covAction, ['push', 'remove', 'deploy_status'], true)) {
|
||||
$dir = rtrim(SCRIPTS_DIR, '/');
|
||||
$script = $dir . '/Fallback/coverage_deploy.sh';
|
||||
$runner = $dir . '/Plugin/unraid/run_job.sh';
|
||||
|
||||
if (!is_file($script)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'coverage_deploy.sh not found on this host']);
|
||||
exit;
|
||||
}
|
||||
|
||||
// Status is read-only and fast enough to answer inline; the two that change the partner are
|
||||
// dispatched to run_job.sh so they get a job record, a log, and a UI surface like every other
|
||||
// long operation here.
|
||||
if ($_covAction === 'deploy_status') {
|
||||
$out = [];
|
||||
exec('timeout 120 /bin/bash ' . escapeshellarg($script) . ' --status 2>&1', $out, $rc);
|
||||
$present = []; $missing = [];
|
||||
foreach ($out as $line) {
|
||||
if (preg_match('/^\s{2}(\S+)\s+MISSING on/', $line, $m)) $missing[] = $m[1];
|
||||
elseif (preg_match('/^\s{2}(\S+)\s+on \S+ \((\w+)\)/', $line, $m)) $present[$m[1]] = $m[2];
|
||||
}
|
||||
echo json_encode([
|
||||
'ok' => true,
|
||||
'present' => $present,
|
||||
'missing' => $missing,
|
||||
// rc 2 means "ran fine, some are missing" — not a failure of the check itself.
|
||||
'checked' => ($rc === 0 || $rc === 2),
|
||||
]);
|
||||
exit;
|
||||
}
|
||||
|
||||
if (!is_file($runner)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'run_job.sh not found on this host']);
|
||||
exit;
|
||||
}
|
||||
$flag = $_covAction === 'push' ? '--push' : '--remove';
|
||||
$stat = '/var/log/varaverk/Fallback/coverage_deploy.json';
|
||||
|
||||
shell_exec('setsid /bin/bash ' . escapeshellarg($runner)
|
||||
. ' ' . escapeshellarg('Fallback/coverage_deploy.sh')
|
||||
. ' ' . escapeshellarg($script)
|
||||
. ' --manual ' . escapeshellarg($flag)
|
||||
. ' >/dev/null 2>&1 </dev/null &');
|
||||
|
||||
// Report what the record says, not that the command was issued — run_job.sh writes its stat
|
||||
// file before running, so a live record is the difference between a job that started and one
|
||||
// refused for already running, or killed by the NORMAL-state gate.
|
||||
for ($i = 0; $i < 12; $i++) {
|
||||
if (is_file($stat)) {
|
||||
$j = json_decode((string)@file_get_contents($stat), true);
|
||||
if (is_array($j) && ($j['status'] ?? '') === 'running' && time() - filemtime($stat) < 60) {
|
||||
echo json_encode(['ok' => true, 'status' => 'running', 'action' => $_covAction]);
|
||||
exit;
|
||||
}
|
||||
}
|
||||
usleep(250000);
|
||||
}
|
||||
echo json_encode(['ok' => false,
|
||||
'error' => 'Job did not report as running — check the Fallback log. It refuses to run unless fallback state is NORMAL.']);
|
||||
exit;
|
||||
}
|
||||
if (($_POST['action'] ?? '') !== 'cover') {
|
||||
echo json_encode(['ok' => false, 'error' => 'Unknown action']);
|
||||
exit;
|
||||
|
||||
@@ -1483,6 +1483,114 @@ function vv_ai_bug_report(array $b): string {
|
||||
// Strictly read-only, and there is no counterpart that changes any of it. Knowing a container is
|
||||
// down is what lets an explanation be about this machine instead of about Unraid in general;
|
||||
// restarting it is a decision that belongs to a person looking at the screen.
|
||||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// Fallback readiness, for the assistant on the Fallback tab.
|
||||
//
|
||||
// Fallback differs from every other subsystem here in one way that shapes this whole function: it
|
||||
// is DORMANT until it isn't. A watchdog leaves strikes and restarts to reason about; fallback
|
||||
// leaves nothing at all until a real outage, so "looks fine" and "would work" are unrelated. On
|
||||
// 2026-08-23 the coverage card showed 12 containers configured and every one of them was absent
|
||||
// from the partner — a failover would have started nothing, and no surface said so.
|
||||
//
|
||||
// So this reports what would ACTUALLY happen, not what is configured to happen, and it is explicit
|
||||
// about the difference between the two.
|
||||
//
|
||||
// Never blocks on the network. Partner presence costs an SSH round trip per container, which is far
|
||||
// too slow for a question already waiting on a model, so it is read from the cache
|
||||
// coverage_deploy.sh --status writes and reported WITH ITS AGE. A stale answer stated as stale is
|
||||
// useful; a stale answer stated as current is the failure this whole feature exists to prevent.
|
||||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
function vv_ai_fallback_state(): string {
|
||||
$me = vv_detect_host();
|
||||
if ($me === '') return '';
|
||||
// vv_detect_host() returns the LOWERCASE slug (host1); the conf keys are uppercase
|
||||
// (FALLBACK_HOST1_TIER1). Building the key from the slug as-is silently matched nothing and
|
||||
// reported "NOTHING is covered" on a host with twelve covered containers — a confidently
|
||||
// wrong answer, which is the one outcome this block must never produce.
|
||||
$ME = strtoupper($me);
|
||||
|
||||
$conf = vv_read_conf_raw('master.conf');
|
||||
if ($conf === '') return '';
|
||||
|
||||
$s = "FALLBACK READINESS (read-only — you cannot change any of it, and you must never tell the "
|
||||
. "operator a failover will work unless the evidence below says so)\n";
|
||||
|
||||
// ── current state ───────────────────────────────────────────────────────────────────────
|
||||
$stateFile = STATE_DIR . '/fallback_state.db';
|
||||
$state = 'unknown'; $since = '';
|
||||
if (is_readable($stateFile)) {
|
||||
$raw = (string) @file_get_contents($stateFile);
|
||||
if (preg_match('/^state=(\S+)/m', $raw, $m)) $state = $m[1];
|
||||
if (preg_match('/^fallback_start=(\d+)/m', $raw, $m) && (int)$m[1] > 0) {
|
||||
$since = ' since ' . date('Y-m-d H:i', (int) $m[1]);
|
||||
}
|
||||
$age = time() - (int) @filemtime($stateFile);
|
||||
// The steady NORMAL path writes nothing, so an old mtime is not staleness — it is quiet.
|
||||
$s .= "- state: $state$since (state file last written "
|
||||
. ($age < 3600 ? round($age / 60) . ' minutes' : round($age / 86400) . ' days') . " ago; "
|
||||
. "the NORMAL path writes nothing, so an old file means nothing has changed)\n";
|
||||
} else {
|
||||
// No file is not the same as not running: fallback.sh writes only on a transition.
|
||||
$live = function_exists('vv_fb_proc') ? (vv_fb_proc('fallback')['running'] ?? false) : false;
|
||||
$s .= $live
|
||||
? "- state: NORMAL (inferred — the daemon is running and has never recorded a transition, "
|
||||
. "so it has written no state file; this is health, not ignorance)\n"
|
||||
: "- state: no state file AND no running daemon — fallback is not operating on this host\n";
|
||||
}
|
||||
|
||||
foreach (['FALLBACK_ENABLED', 'FALLBACK_RSYNC_ENABLED'] as $k) {
|
||||
if (preg_match('/^\s*' . $k . '\s*=\s*"?(\w+)"?/m', $conf, $m)) {
|
||||
$s .= "- $k: {$m[1]}"
|
||||
. ($k === 'FALLBACK_RSYNC_ENABLED' && $m[1] !== 'true'
|
||||
? " <- handback writeback is OFF: anything the partner writes while covering "
|
||||
. "for this host never comes home\n" : "\n");
|
||||
}
|
||||
}
|
||||
|
||||
// ── coverage, tier by tier, with the real delays ─────────────────────────────────────────
|
||||
$hostConf = vv_read_conf_raw($me . '.conf');
|
||||
$covered = [];
|
||||
for ($t = 1; $t <= 4; $t++) {
|
||||
$names = vv_parse_conf_list($hostConf, "FALLBACK_{$ME}_TIER{$t}");
|
||||
if (!$names) continue;
|
||||
$delay = '';
|
||||
if ($t > 1 && preg_match('/^\s*' . $ME . '_TIER' . $t . '_DELAY\s*=\s*"?(\d+)/m', $hostConf, $m)) {
|
||||
$delay = " after {$m[1]} minutes";
|
||||
}
|
||||
$s .= "- tier $t" . ($t === 1 ? ' (immediate)' : $delay) . ': ' . implode(', ', $names) . "\n";
|
||||
foreach ($names as $n) $covered[] = $n;
|
||||
}
|
||||
if (!$covered) {
|
||||
$s .= "- coverage: NOTHING is covered — a failover would start no containers at all\n";
|
||||
return $s . "\n";
|
||||
}
|
||||
|
||||
// ── does the partner actually have them ─────────────────────────────────────────────────
|
||||
$cache = '/tmp/varaverk/api/fallback_presence.json';
|
||||
if (is_readable($cache)) {
|
||||
$j = json_decode((string) @file_get_contents($cache), true);
|
||||
$age = time() - (int) @filemtime($cache);
|
||||
$miss = (array) ($j['missing'] ?? []);
|
||||
$have = array_keys((array) ($j['present'] ?? []));
|
||||
$when = $age < 3600 ? round($age / 60) . ' minutes ago' : round($age / 3600) . ' hours ago';
|
||||
if ($miss) {
|
||||
$s .= "- ON THE PARTNER (checked $when): " . count($miss) . ' of ' . count($covered)
|
||||
. " covered container(s) DO NOT EXIST there: " . implode(', ', $miss) . "\n"
|
||||
. " fallback.sh starts a covered container with `docker start`; it never creates one, "
|
||||
. "so each of those would fail during a real outage. Push them from the Fallback "
|
||||
. "coverage card.\n";
|
||||
} else {
|
||||
$s .= "- ON THE PARTNER (checked $when): all " . count($have)
|
||||
. " covered container(s) exist there\n";
|
||||
}
|
||||
} else {
|
||||
$s .= "- ON THE PARTNER: not checked. Say so plainly — whether a failover would actually "
|
||||
. "start anything is UNKNOWN until the coverage card's presence check runs.\n";
|
||||
}
|
||||
|
||||
return $s . "\n";
|
||||
}
|
||||
|
||||
function vv_ai_system_state(): string {
|
||||
$p = '/tmp/varaverk/api/monitor.json';
|
||||
if (!is_readable($p)) return '';
|
||||
|
||||
@@ -116,7 +116,7 @@ const VV_AI_PROFILES_DEF = [
|
||||
// system_state is read-only and shared with repair. Both need to know a container is down
|
||||
// or a pool is full to explain anything about this machine rather than about Unraid in
|
||||
// general; neither gets a way to act on it, and only repair can change a setting.
|
||||
'caps' => ['retrieve', 'health', 'system_state', 'run_evidence', 'scoped_log',
|
||||
'caps' => ['retrieve', 'health', 'system_state', 'fallback_state', 'run_evidence', 'scoped_log',
|
||||
'incidents', 'conf_lookup', 'file_bugs'],
|
||||
],
|
||||
// The only profile that may change a setting, and the only one not offered as a button.
|
||||
@@ -136,7 +136,7 @@ const VV_AI_PROFILES_DEF = [
|
||||
'hint' => 'Works through a finding with you, and can apply a fix you approve.',
|
||||
'turns' => 3,
|
||||
'ui' => false,
|
||||
'caps' => ['retrieve', 'health', 'system_state', 'run_evidence', 'scoped_log', 'incidents',
|
||||
'caps' => ['retrieve', 'health', 'system_state', 'fallback_state', 'run_evidence', 'scoped_log', 'incidents',
|
||||
'conf_lookup', 'conf_write', 'probe', 'file_findings', 'phrasebook', 'past_fixes'],
|
||||
],
|
||||
];
|
||||
@@ -148,6 +148,7 @@ const VV_AI_CAP_MEANING = [
|
||||
'kind_filter' => 'the retrieval kind filter the page exposes',
|
||||
'health' => 'live health sweep measured at question time — the AI subsystem only',
|
||||
'system_state' => 'read-only view of the machine: hardware, containers, pools, array, UPS',
|
||||
'fallback_state' => 'whether a failover would actually work: state, tiers, and whether the partner really has the covered containers',
|
||||
'run_evidence' => 'run record and log tail for a script named in the question',
|
||||
'scoped_log' => 'log tail for whatever the operator currently has open',
|
||||
'incidents' => 'operator-written history of what previously went wrong with this thing',
|
||||
|
||||
@@ -73,6 +73,8 @@ function vv_fb_parse_state(string $text): array {
|
||||
'tier4_started' => false,
|
||||
'partnership_suspended' => false,
|
||||
'partner_lost_at' => 0,
|
||||
// Set only when the verdict came from the daemon rather than the file — see vv_fb_all().
|
||||
'inferred' => false,
|
||||
];
|
||||
foreach (explode("\n", $text) as $line) {
|
||||
$line = trim($line);
|
||||
@@ -313,6 +315,29 @@ function vv_fb_all(): array {
|
||||
: vv_fb_remote_dryrun_state($ip, $mySshKey, (int)$proc['pid']);
|
||||
}
|
||||
|
||||
// ── A live daemon that has simply never transitioned ────────────────────────────────
|
||||
// fallback.sh writes its state file ONLY on a transition; the steady NORMAL path writes
|
||||
// nothing at all. So a node that has run cleanly since it was built has no file, and
|
||||
// reading the file alone reports it as UNKNOWN — the same verdict given to a node whose
|
||||
// daemon is dead. Those are opposite conditions and they were rendered identically.
|
||||
//
|
||||
// Observed on HOST2 2026-08-23: daemon live (pid 2208657, valid lock), tailscale and ssh
|
||||
// both fine, no state file, card said UNKNOWN.
|
||||
//
|
||||
// The rule this page is built on — never claim healthy for a host you cannot verify — is
|
||||
// kept: this host CAN be verified, just not from the file that was being consulted. The
|
||||
// daemon holding a live lock is the evidence. reach['state_file'] still reports false,
|
||||
// because there genuinely is no file; 'inferred' says where the verdict came from instead.
|
||||
// Captured before the inference below rewrites it: reach[state_file] must keep answering
|
||||
// "was there a file", not "do we have a verdict". Deriving it after inference made the
|
||||
// card claim a state file existed on a host that has none.
|
||||
$hadStateFile = ($state['state'] ?? 'UNKNOWN') !== 'UNKNOWN';
|
||||
|
||||
if (($state['state'] ?? 'UNKNOWN') === 'UNKNOWN' && ($proc['running'] ?? null) === true) {
|
||||
$state['state'] = 'NORMAL';
|
||||
$state['inferred'] = true;
|
||||
}
|
||||
|
||||
// How fresh the state actually is. The daemon rewrites its file every check interval, so
|
||||
// an age far past that interval means it is wedged even while the process still exists.
|
||||
$stateAge = null;
|
||||
@@ -326,7 +351,7 @@ function vv_fb_all(): array {
|
||||
'tailscale' => $isMe ? true : ($ts['online'] === true),
|
||||
'ip' => $isMe ? null : $ip,
|
||||
'ssh' => $isMe ? true : ($running !== [] || ($proc['running'] !== null)),
|
||||
'state_file' => ($state['state'] ?? 'UNKNOWN') !== 'UNKNOWN',
|
||||
'state_file' => $hadStateFile,
|
||||
];
|
||||
|
||||
$nodes[] = [
|
||||
|
||||
@@ -64,6 +64,28 @@ if (vv_ai_ui_on()) vv_ai_chat_assets();
|
||||
border-radius:4px;padding:6px 9px;margin-bottom:8px; }
|
||||
.vv-fb-tierhdr-t { font-size:12px;font-weight:700;color:#ffb74d;letter-spacing:.01em; }
|
||||
.vv-fb-tierhdr-s { font-size:10px;color:#7a6038; }
|
||||
/* ── Failover readiness ──────────────────────────────────────────────────────────────────────
|
||||
Every other card on this page shows what is CONFIGURED. This one shows what would actually
|
||||
happen, which on 2026-08-23 turned out to be a different thing entirely — twelve containers
|
||||
configured, none of them present on the partner, and no surface said so.
|
||||
Verdict colour always ships beside a word, never alone. */
|
||||
.vv-fb-rd { display:flex;flex-direction:column;gap:4px; }
|
||||
.vv-fb-rdrow { display:flex;align-items:center;gap:9px;padding:5px 8px;border-radius:3px;
|
||||
background:#0d0d0d;border:1px solid #161616;border-left:3px solid var(--rv,#333); }
|
||||
.vv-fb-rdrow.ok { --rv:#4caf50; }
|
||||
.vv-fb-rdrow.warn { --rv:#ffb74d; }
|
||||
.vv-fb-rdrow.fail { --rv:#ef5350; }
|
||||
.vv-fb-rdrow.unknown { --rv:#5a7a8a; }
|
||||
.vv-fb-rdv { font-size:9px;font-weight:700;letter-spacing:.05em;text-transform:uppercase;
|
||||
min-width:56px;color:var(--rv,#666); }
|
||||
.vv-fb-rdl { font-size:12px;color:#b8b8b8;min-width:190px; }
|
||||
.vv-fb-rdd { font-size:11px;color:#5a5a5a;flex:1;min-width:0;overflow:hidden;
|
||||
text-overflow:ellipsis;white-space:nowrap; }
|
||||
.vv-fb-rdwhy { font-size:10px;padding:2px 8px;border-radius:3px;cursor:pointer;
|
||||
background:#0e1a2a;color:#7ab;border:1px solid #1e3a5a;white-space:nowrap; }
|
||||
.vv-fb-rdwhy:hover { background:#12233a; }
|
||||
.vv-fb-rdsum { font-size:11px;font-weight:600;margin-bottom:7px; }
|
||||
|
||||
/* ── Fallback coverage ── */
|
||||
/* One continuum, worst outcome to best: never comes back → 24h → 12h → 4h → immediate → never
|
||||
goes down at all. The colour answers "how long am I without this if the partner takes over",
|
||||
@@ -267,6 +289,18 @@ if (vv_ai_ui_on()) vv_ai_chat_assets();
|
||||
<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">Loading…</div>
|
||||
</div>
|
||||
|
||||
<!-- Failover readiness — deterministic checks. The assistant EXPLAINS these rows and never
|
||||
produces them: a model must not be the thing that says a failover will work. -->
|
||||
<div class="vv-card" id="vv-fb-readiness" style="margin-bottom:12px;">
|
||||
<div style="display:flex;align-items:center;gap:8px;margin-bottom:8px;">
|
||||
<h3 style="margin:0;">Failover readiness</h3>
|
||||
<span style="font-size:10px;color:#444;">would a failover actually work right now</span>
|
||||
<span style="flex:1;"></span>
|
||||
<button class="vv-fb-save-btn" onclick="vvFbReadiness(true)" title="Re-run the checks">↻</button>
|
||||
</div>
|
||||
<div class="vv-fb-rdsum" id="vv-fb-rdsum">checking…</div>
|
||||
<div class="vv-fb-rd" id="vv-fb-rdrows"></div>
|
||||
</div>
|
||||
<!-- Fallback coverage — this host's own tiers -->
|
||||
<!--
|
||||
Originally on the Partnership page (e8ee5b0), removed the same day in 1a836da because it
|
||||
@@ -289,7 +323,23 @@ if (vv_ai_ui_on()) vv_ai_chat_assets();
|
||||
To change what <span id="vv-fb-cov-partner" style="color:#666;">the partner</span> hands to us, open this page there.
|
||||
</div>
|
||||
<div id="vv-fb-cov-body" style="color:#444;font-size:12px;">Loading…</div>
|
||||
<div style="display:flex;justify-content:flex-end;align-items:center;gap:10px;margin-top:10px;padding-top:8px;border-top:1px solid #1e1e1e;">
|
||||
<!-- Deploy row, deliberately separate from Save. Save writes the tier list; these two change
|
||||
what the partner physically holds. fallback.sh starts a covered container with
|
||||
`docker start`, which fails unless it was built there first — so a coverage list the
|
||||
partner has never been sent is a promise nothing can keep, and this row is where that
|
||||
is made visible and fixed. -->
|
||||
<div id="vv-fb-cov-deploy" style="display:flex;align-items:center;gap:8px;flex-wrap:wrap;margin-top:10px;padding-top:8px;border-top:1px solid #1e1e1e;">
|
||||
<span style="font-size:10px;color:#5a5a5a;text-transform:uppercase;letter-spacing:.04em;">On partner</span>
|
||||
<span id="vv-fb-cov-presence" style="font-size:11px;color:#444;">checking…</span>
|
||||
<span style="flex:1;"></span>
|
||||
<span id="vv-fb-cov-dfb" style="font-size:11px;"></span>
|
||||
<button class="vv-fb-save-btn" id="vv-fb-cov-push" onclick="vvFbCovDeploy('push')"
|
||||
title="Build every covered container on the partner, left stopped, so a failover can start them">Push to partner</button>
|
||||
<button class="vv-fb-save-btn" id="vv-fb-cov-rm" onclick="vvFbCovDeploy('remove')"
|
||||
style="background:#1a1208;color:#c88;border-color:#3a2a1a;"
|
||||
title="Stop and remove these containers on the partner, and delete their appdata there. Not reversible.">Remove from partner</button>
|
||||
</div>
|
||||
<div style="display:flex;justify-content:flex-end;align-items:center;gap:10px;margin-top:8px;">
|
||||
<span id="vv-fb-cov-fb" style="font-size:11px;"></span>
|
||||
<button class="vv-fb-save-btn" id="vv-fb-cov-save" onclick="vvFbCovSave()">Save coverage</button>
|
||||
</div>
|
||||
@@ -654,13 +704,14 @@ function _nodeCard(node, data) {
|
||||
${node.is_me ? '<span class="vv-fb-usbadge">US</span>' : ''}
|
||||
<span style="flex:1"></span>
|
||||
${(node.is_me && node.proc && node.proc.running && node.proc.mode === 'live') ? _ptStatus(st) : ''}
|
||||
${_stateBadge(shown)}${dryRun ? '<span class="vv-fb-leg" style="margin-left:4px;">preview</span>' : ''}
|
||||
${_stateBadge(shown)}${dryRun ? '<span class="vv-fb-leg" style="margin-left:4px;">preview</span>' : ''}${(st.inferred && !dryRun) ? '<span class="vv-fb-leg" style="margin-left:4px;" title="Inferred from the running daemon rather than read from a state file — this node has never transitioned.">inferred</span>' : ''}
|
||||
</div>
|
||||
|
||||
<div class="vv-fb-legs">
|
||||
${_leg(reach.tailscale, 'tailscale')}
|
||||
${_leg(node.is_me ? true : reach.ssh, 'ssh')}
|
||||
${_leg(reach.state_file, 'state file')}
|
||||
${st.inferred ? '<span class="vv-fb-leg" title="fallback.sh writes its state file only on a transition, so a node that has run cleanly since it was built has none. The verdict comes from the live daemon holding a valid lock.">· never transitioned</span>' : ''}
|
||||
${node.ts_ip ? `<span class="vv-fb-leg">${vvEscHtml(node.ts_ip)}</span>` : ''}
|
||||
</div>
|
||||
|
||||
@@ -1081,9 +1132,174 @@ window.vvFbCovSave = async function () {
|
||||
fbEl.style.color = '#ef5350'; fbEl.textContent = 'Request failed: ' + e;
|
||||
});
|
||||
};
|
||||
|
||||
function _vvFbPartnerName() {
|
||||
const el = document.getElementById('vv-fb-cov-partner');
|
||||
const t = el ? el.textContent.trim() : '';
|
||||
return (t && t !== 'the partner') ? t : 'the partner';
|
||||
}
|
||||
|
||||
// ── Deploy: what the partner actually holds ───────────────────────────────────────────────────
|
||||
// Presence is read from the partner, never inferred from the tier list. The whole point of this
|
||||
// row is that the two disagree — coverage said 12 containers, the partner had none of them.
|
||||
window.vvFbCovPresence = async function () {
|
||||
const el = document.getElementById('vv-fb-cov-presence');
|
||||
if (!el) return;
|
||||
try {
|
||||
const fd = new URLSearchParams({ action: 'deploy_status' });
|
||||
const d = await (await fetch('/plugins/varaverk/api/fallback_coverage.php',
|
||||
{ method: 'POST', body: fd })).json();
|
||||
if (!d.ok || !d.checked) { el.style.color = '#a05a2c'; el.textContent = d.error || 'could not check'; return; }
|
||||
const have = Object.keys(d.present || {}).length, miss = (d.missing || []).length;
|
||||
if (miss === 0 && have === 0) { el.style.color = '#444'; el.textContent = 'nothing covered'; }
|
||||
else if (miss === 0) { el.style.color = '#4caf50'; el.textContent = `all ${have} present`; }
|
||||
else {
|
||||
el.style.color = '#ef5350';
|
||||
// Named, not just counted: "3 missing" is a number, the names are what you act on.
|
||||
el.textContent = `${miss} missing — ${(d.missing||[]).slice(0,3).join(', ')}${miss>3?` +${miss-3}`:''}`;
|
||||
}
|
||||
} catch (e) { el.style.color = '#a05a2c'; el.textContent = 'check failed'; }
|
||||
};
|
||||
|
||||
window.vvFbCovDeploy = async function (which) {
|
||||
const push = which === 'push';
|
||||
const fbEl = document.getElementById('vv-fb-cov-dfb');
|
||||
const btn = document.getElementById(push ? 'vv-fb-cov-push' : 'vv-fb-cov-rm');
|
||||
const n = _vvFbCov ? Object.keys(_vvFbCov.cover).length : 0;
|
||||
|
||||
const msg = push
|
||||
? `Build ${n} container${n!==1?'s':''} on ${_vvFbPartnerName()}?\n\n`
|
||||
+ 'Each is created and left STOPPED so a failover can start it. Nothing starts running now.\n\n'
|
||||
+ 'Save first if you have unsaved changes — this pushes what is in the conf, not what is on screen.'
|
||||
: `Stop and remove ${n} container${n!==1?'s':''} on ${_vvFbPartnerName()} AND DELETE THEIR APPDATA?\n\n`
|
||||
+ 'Not reversible. Only paths under /mnt/*/appdata* are touched; a bind of the appdata root is refused.\n\n'
|
||||
+ 'If the partner has ever covered for this host, what it holds may be the NEWER copy — the one a '
|
||||
+ 'handback rsyncs home. Fallback state is NORMAL, so nothing is failing over right now, but a '
|
||||
+ 'handback that partly failed would not show up here.\n\n'
|
||||
+ 'Coverage stays as configured, so a later Push rebuilds the containers from scratch.';
|
||||
if (!await vvConfirm(msg, { title: push ? 'Push to partner' : 'Remove from partner',
|
||||
confirmText: push ? 'Push' : 'Remove' })) return;
|
||||
|
||||
btn.disabled = true; const label = btn.textContent; btn.textContent = push ? 'Pushing…' : 'Removing…';
|
||||
fbEl.style.color = '#7ab'; fbEl.textContent = 'job started…';
|
||||
try {
|
||||
const fd = new URLSearchParams({ action: which });
|
||||
const d = await (await fetch('/plugins/varaverk/api/fallback_coverage.php',
|
||||
{ method: 'POST', body: fd })).json();
|
||||
if (!d.ok) { fbEl.style.color = '#ef5350'; fbEl.textContent = d.error || 'Failed'; }
|
||||
else {
|
||||
// The job runs past this response. Re-checking presence is the only honest completion
|
||||
// signal available here, so poll it rather than claiming success on dispatch.
|
||||
fbEl.style.color = '#7ab'; fbEl.textContent = 'running — see the Fallback log';
|
||||
let ticks = 0;
|
||||
const t = setInterval(async () => {
|
||||
await vvFbCovPresence();
|
||||
// Presence just changed, so the readiness verdict that depends on it is stale.
|
||||
vvFbReadiness(false);
|
||||
if (++ticks >= 20) { clearInterval(t); fbEl.textContent = ''; }
|
||||
}, 6000);
|
||||
}
|
||||
} catch (e) {
|
||||
fbEl.style.color = '#ef5350'; fbEl.textContent = 'Request failed: ' + e;
|
||||
}
|
||||
btn.disabled = false; btn.textContent = label;
|
||||
};
|
||||
// ── Assistant ────────────────────────────────────────────────────────────────────────────────
|
||||
// vv_ai_chat_markup() above emits the boxes and nothing else — no <script>, no init. Without this
|
||||
// block the card renders looking complete and dies on the first click with VvAiChat undefined.
|
||||
// Fallback was the only one of seven pages mounting a dock and never instantiating it, which is
|
||||
// exactly the failure watchdog.php warns about in its own comment.
|
||||
//
|
||||
// troubleshoot, not varaverk: the placeholder invites "what would the partner start if this host
|
||||
// went dark", and a docs-only profile cannot reach live state to answer it. On 2026-08-23 the
|
||||
// documented answer would also have been wrong — coverage listed 12 containers and none of them
|
||||
// existed on the partner.
|
||||
let vvFbChat = null;
|
||||
let vvFbScope = 'Fallback';
|
||||
|
||||
if (typeof VvAiChat === 'function' && document.getElementById('vv-fb-ai-chat')) {
|
||||
vvFbChat = VvAiChat({
|
||||
prefix: 'vv-fb-ai',
|
||||
profile: 'troubleshoot',
|
||||
scopeLabel: 'Fallback',
|
||||
// Read at send time rather than captured — a Why? retargets the scope and sends from the
|
||||
// same click.
|
||||
scope: () => vvFbScope,
|
||||
// Pinned for the same reason Monitor pins its own: without it the card resumes whatever
|
||||
// thread was last touched anywhere, landing this tab mid-conversation under a profile it
|
||||
// never offers.
|
||||
resumeProfile: 'troubleshoot',
|
||||
think: p => p === 'troubleshoot',
|
||||
empty: 'Ask about fallback — why a tier has not fired, what the partner would actually '
|
||||
+ 'start if this host went dark, whether a stale state file matters.',
|
||||
});
|
||||
}
|
||||
|
||||
// Retarget the assistant at one thing on the page, then ask about it — same shape as the Watchdog
|
||||
// Why? buttons, and deliberately using the component's real API. There is no ask(): it is
|
||||
// retarget() + set the input + send(), and calling a method that does not exist would fail
|
||||
// silently on click, which is the bug this page already had once.
|
||||
window.vvFbWhy = function (label, question, scope) {
|
||||
if (!vvFbChat || vvFbChat.busy()) return;
|
||||
// troubleshoot, never repair. This page arms and disarms failover; repair is the one profile
|
||||
// that can write conf, and a chat box is the wrong place to do that from.
|
||||
vvFbScope = scope || label;
|
||||
vvFbChat.retarget('troubleshoot', label, 'now looking at ' + label);
|
||||
const input = document.getElementById('vv-fb-ai-input');
|
||||
if (input) input.value = question;
|
||||
vvFbChat.send();
|
||||
const card = document.getElementById('vv-fb-ai-card');
|
||||
if (card) card.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
|
||||
};
|
||||
|
||||
// ── Failover readiness ───────────────────────────────────────────────────────────────────────
|
||||
// Rendered from the endpoint's verdicts verbatim. Nothing here decides anything — if a row says
|
||||
// fail, it is because a check failed, not because the page inferred it.
|
||||
window.vvFbReadiness = async function (force) {
|
||||
const sum = document.getElementById('vv-fb-rdsum');
|
||||
const rows = document.getElementById('vv-fb-rdrows');
|
||||
if (!sum || !rows) return;
|
||||
if (force) { sum.textContent = 'checking…'; sum.style.color = '#7ab'; }
|
||||
|
||||
try {
|
||||
const d = await (await fetch('/plugins/varaverk/api/fallback_coverage.php',
|
||||
{ method: 'POST', body: new URLSearchParams({ action: 'readiness' }) })).json();
|
||||
if (!d.ok) throw new Error(d.error || 'no verdict');
|
||||
|
||||
const tone = { ok: '#4caf50', warn: '#ffb74d', fail: '#ef5350', unknown: '#5a7a8a' };
|
||||
sum.style.color = tone[d.verdict] || '#888';
|
||||
sum.textContent = d.summary;
|
||||
|
||||
rows.innerHTML = (d.rows || []).map(r =>
|
||||
`<div class="vv-fb-rdrow ${vvEscAttr(r.verdict)}">`
|
||||
+ `<span class="vv-fb-rdv">${vvEscHtml(r.verdict)}</span>`
|
||||
+ `<span class="vv-fb-rdl">${vvEscHtml(r.label)}</span>`
|
||||
+ `<span class="vv-fb-rdd" title="${vvEscAttr(r.detail)}">${vvEscHtml(r.detail)}</span>`
|
||||
+ (r.ask ? `<span class="vv-fb-rdwhy" data-ask="${vvEscAttr(r.ask)}" `
|
||||
+ `data-label="${vvEscAttr(r.label)}">Why?</span>` : '')
|
||||
+ `</div>`).join('');
|
||||
} catch (e) {
|
||||
sum.style.color = '#ef5350';
|
||||
sum.textContent = 'Could not run the checks — ' + e;
|
||||
rows.innerHTML = '';
|
||||
}
|
||||
};
|
||||
|
||||
// Delegated: the rows are rebuilt on every refresh, so per-node handlers would leak. Neither the
|
||||
// question nor the label is interpolated into an onclick — vvEscHtml does not escape quotes.
|
||||
(function () {
|
||||
const host = document.getElementById('vv-fb-rdrows');
|
||||
if (host) host.addEventListener('click', ev => {
|
||||
const b = ev.target.closest('.vv-fb-rdwhy');
|
||||
if (b) vvFbWhy(b.dataset.label || 'Fallback readiness', b.dataset.ask || 'What does this mean?');
|
||||
});
|
||||
})();
|
||||
|
||||
vvFbLoad();
|
||||
setInterval(vvFbLoad, 30000);
|
||||
vvFbCovLoad(); // once — this is an editor, not a monitor; polling would fight the operator
|
||||
vvFbCovPresence();
|
||||
vvFbReadiness(false);
|
||||
|
||||
})();
|
||||
</script>
|
||||
|
||||
Reference in New Issue
Block a user