Make fallback coverage something that can actually happen, and say so on the page
fallback.sh starts covered containers with docker start and never creates them, so a coverage list the partner has never been sent is a promise nothing can keep — all twelve were missing. Adds the push and remove paths, a readiness card that checks rather than infers, and the fallback state the assistant needs to answer for it.
This commit is contained in:
Gmer4Lfe
@@ -59,6 +59,7 @@
|
||||
// include/config.php vv_detect_host(), vv_read_conf_raw(), vv_push_master_conf()
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/fallback.php'; // vv_fb_proc()
|
||||
require_once dirname(__DIR__) . '/include/confform.php';
|
||||
require_once dirname(__DIR__) . '/include/common.php';
|
||||
|
||||
@@ -143,6 +144,191 @@ if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
||||
}
|
||||
|
||||
// ── Write ────────────────────────────────────────────────────────────────────────────────────
|
||||
// ── Readiness: would a failover actually work right now ─────────────────────────────────────
|
||||
// Every row is a deterministic check with a stated basis. The assistant on this page EXPLAINS
|
||||
// these rows; it never produces them. A model must not be the thing that says failover is ready —
|
||||
// that is precisely the class of answer this codebase keeps finding to be confidently wrong, and
|
||||
// on 2026-08-23 the coverage card itself was the confidently wrong surface: 12 containers listed,
|
||||
// none of them present on the partner.
|
||||
//
|
||||
// Rows are ordered by what breaks first, not by severity, so reading top to bottom follows the
|
||||
// order a real outage would hit them.
|
||||
if (($_POST['action'] ?? '') === 'readiness') {
|
||||
$rows = [];
|
||||
$add = function (string $id, string $label, string $verdict, string $detail, string $ask = '')
|
||||
use (&$rows) {
|
||||
// verdict: ok | warn | fail | unknown — unknown is never dressed up as ok
|
||||
$rows[] = ['id' => $id, 'label' => $label, 'verdict' => $verdict,
|
||||
'detail' => $detail, 'ask' => $ask];
|
||||
};
|
||||
|
||||
$me = vv_detect_host();
|
||||
$ME = strtoupper($me);
|
||||
$conf = vv_read_conf_raw('master.conf');
|
||||
$hc = vv_read_conf_raw($me . '.conf');
|
||||
|
||||
// 1. is fallback even armed
|
||||
$fbEnabled = preg_match('/^\s*FALLBACK_ENABLED\s*=\s*"?(\w+)/m', $conf, $m) ? $m[1] : 'unset';
|
||||
$add('enabled', 'Fallback armed',
|
||||
$fbEnabled === 'true' ? 'ok' : 'fail',
|
||||
'FALLBACK_ENABLED=' . $fbEnabled,
|
||||
'FALLBACK_ENABLED is ' . $fbEnabled . ' — what does that mean for a real outage?');
|
||||
|
||||
// 2. current state — anything but NORMAL means it is already doing something
|
||||
$stateFile = STATE_DIR . '/fallback_state.db';
|
||||
$state = 'unknown';
|
||||
if (is_readable($stateFile) && preg_match('/^state=(\S+)/m', (string)@file_get_contents($stateFile), $m)) {
|
||||
$state = $m[1];
|
||||
}
|
||||
// Same rule the node cards use: no state file plus a live daemon means the node has simply
|
||||
// never transitioned, which is health, not ignorance. Reading the file alone gives a healthy
|
||||
// node the same verdict as one whose daemon is dead.
|
||||
$daemon = function_exists('vv_fb_proc') ? (vv_fb_proc('fallback')['running'] ?? false) : false;
|
||||
$inferred = false;
|
||||
if ($state === 'unknown' && $daemon) { $state = 'NORMAL'; $inferred = true; }
|
||||
|
||||
$add('state', 'State is NORMAL',
|
||||
$state === 'NORMAL' ? 'ok' : ($state === 'unknown' ? 'unknown' : 'warn'),
|
||||
'state=' . $state . ($inferred ? ' (from the live daemon — never transitioned)' : ''),
|
||||
'Fallback state is ' . $state . '. What does that mean and what should I check?');
|
||||
|
||||
// 3. coverage configured at all
|
||||
$covered = [];
|
||||
for ($t = 1; $t <= 4; $t++) {
|
||||
foreach (vv_parse_conf_list($hc, "FALLBACK_{$ME}_TIER{$t}") as $c) $covered[] = $c;
|
||||
}
|
||||
$add('coverage', 'Containers are covered',
|
||||
$covered ? 'ok' : 'fail',
|
||||
$covered ? count($covered) . ' container(s) across the tiers' : 'no containers in any tier',
|
||||
$covered
|
||||
? 'Walk me through what happens if this host goes dark right now, tier by tier, with the delays.'
|
||||
: 'Nothing is listed in my fallback tiers — what would happen if this host went dark?');
|
||||
|
||||
// 4. THE one that was silently false — does the partner actually hold them
|
||||
$cache = (defined('VV_CACHE_ROOT') ? VV_CACHE_ROOT : '/tmp/varaverk') . '/api/fallback_presence.json';
|
||||
if (!is_readable($cache)) {
|
||||
$add('present', 'Partner has the containers', 'unknown',
|
||||
'never checked — run the presence check',
|
||||
'How do I find out whether the partner actually has my covered containers?');
|
||||
} else {
|
||||
$j = json_decode((string)@file_get_contents($cache), true);
|
||||
$miss = (array)($j['missing'] ?? []);
|
||||
$age = time() - (int)@filemtime($cache);
|
||||
$when = $age < 3600 ? round($age / 60) . 'm ago' : round($age / 3600) . 'h ago';
|
||||
$add('present', 'Partner has the containers',
|
||||
$miss ? 'fail' : 'ok',
|
||||
$miss ? count($miss) . ' of ' . count($covered) . ' missing (' . $when . '): '
|
||||
. implode(', ', array_slice($miss, 0, 4)) . (count($miss) > 4 ? '…' : '')
|
||||
: 'all ' . count($covered) . ' present (' . $when . ')',
|
||||
$miss ? 'The partner is missing ' . implode(', ', array_slice($miss, 0, 6))
|
||||
. '. What happens during a failover, and how do I fix it?' : '');
|
||||
}
|
||||
|
||||
// 5. host-specific networks that cannot be recreated on the partner
|
||||
$wg = [];
|
||||
foreach ($covered as $c) {
|
||||
foreach (glob('/boot/config/plugins/dockerMan/templates-user/*.xml') as $x) {
|
||||
$t = @file_get_contents($x);
|
||||
if ($t === false || strpos($t, "<Name>$c</Name>") === false) continue;
|
||||
if (preg_match('~<Network>(wg\d+)</Network>~', $t, $m)) $wg[] = "$c ({$m[1]})";
|
||||
break;
|
||||
}
|
||||
}
|
||||
if ($wg) {
|
||||
$add('wgnet', 'No tunnel-bound networks', 'warn',
|
||||
implode(', ', $wg),
|
||||
'Some covered containers use a WireGuard-backed network. Why can that not move to the partner?');
|
||||
}
|
||||
|
||||
// 6. handback writeback — invisible until the day it matters
|
||||
$wb = preg_match('/^\s*FALLBACK_RSYNC_ENABLED\s*=\s*"?(\w+)/m', $conf, $m) ? $m[1] : 'unset';
|
||||
$add('writeback', 'Handback writeback', $wb === 'true' ? 'ok' : 'warn',
|
||||
'FALLBACK_RSYNC_ENABLED=' . $wb,
|
||||
'FALLBACK_RSYNC_ENABLED is ' . $wb . ' — what do I lose on handback?');
|
||||
|
||||
// Overall verdict is the worst row, never an average. One failed check is a failed failover.
|
||||
$order = ['ok' => 0, 'warn' => 1, 'unknown' => 2, 'fail' => 3];
|
||||
$worst = 'ok';
|
||||
foreach ($rows as $r) if ($order[$r['verdict']] > $order[$worst]) $worst = $r['verdict'];
|
||||
|
||||
echo json_encode(['ok' => true, 'verdict' => $worst, 'rows' => $rows,
|
||||
'summary' => $worst === 'ok'
|
||||
? 'Every check passed'
|
||||
: ($worst === 'fail' ? 'A failover would NOT work as configured'
|
||||
: 'Failover is configured but something needs a look')]);
|
||||
exit;
|
||||
}
|
||||
// ── Push / remove / status: what the PARTNER actually holds ──────────────────────────────────
|
||||
// Coverage names a container; fallback.sh starts it with `docker start`, which fails unless the
|
||||
// partner already has it built. Measured 2026-08-23: 12 of 12 covered containers were absent from
|
||||
// the partner, so every tier would have failed on the first real outage. These three actions are
|
||||
// how the card closes and inspects that gap.
|
||||
//
|
||||
// Deliberately NOT folded into `cover`. Saving a tier list is a cheap, reversible config write;
|
||||
// deploying a dozen containers onto another machine is neither, and a stray click should not be
|
||||
// able to do it.
|
||||
$_covAction = $_POST['action'] ?? '';
|
||||
if (in_array($_covAction, ['push', 'remove', 'deploy_status'], true)) {
|
||||
$dir = rtrim(SCRIPTS_DIR, '/');
|
||||
$script = $dir . '/Fallback/coverage_deploy.sh';
|
||||
$runner = $dir . '/Plugin/unraid/run_job.sh';
|
||||
|
||||
if (!is_file($script)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'coverage_deploy.sh not found on this host']);
|
||||
exit;
|
||||
}
|
||||
|
||||
// Status is read-only and fast enough to answer inline; the two that change the partner are
|
||||
// dispatched to run_job.sh so they get a job record, a log, and a UI surface like every other
|
||||
// long operation here.
|
||||
if ($_covAction === 'deploy_status') {
|
||||
$out = [];
|
||||
exec('timeout 120 /bin/bash ' . escapeshellarg($script) . ' --status 2>&1', $out, $rc);
|
||||
$present = []; $missing = [];
|
||||
foreach ($out as $line) {
|
||||
if (preg_match('/^\s{2}(\S+)\s+MISSING on/', $line, $m)) $missing[] = $m[1];
|
||||
elseif (preg_match('/^\s{2}(\S+)\s+on \S+ \((\w+)\)/', $line, $m)) $present[$m[1]] = $m[2];
|
||||
}
|
||||
echo json_encode([
|
||||
'ok' => true,
|
||||
'present' => $present,
|
||||
'missing' => $missing,
|
||||
// rc 2 means "ran fine, some are missing" — not a failure of the check itself.
|
||||
'checked' => ($rc === 0 || $rc === 2),
|
||||
]);
|
||||
exit;
|
||||
}
|
||||
|
||||
if (!is_file($runner)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'run_job.sh not found on this host']);
|
||||
exit;
|
||||
}
|
||||
$flag = $_covAction === 'push' ? '--push' : '--remove';
|
||||
$stat = '/var/log/varaverk/Fallback/coverage_deploy.json';
|
||||
|
||||
shell_exec('setsid /bin/bash ' . escapeshellarg($runner)
|
||||
. ' ' . escapeshellarg('Fallback/coverage_deploy.sh')
|
||||
. ' ' . escapeshellarg($script)
|
||||
. ' --manual ' . escapeshellarg($flag)
|
||||
. ' >/dev/null 2>&1 </dev/null &');
|
||||
|
||||
// Report what the record says, not that the command was issued — run_job.sh writes its stat
|
||||
// file before running, so a live record is the difference between a job that started and one
|
||||
// refused for already running, or killed by the NORMAL-state gate.
|
||||
for ($i = 0; $i < 12; $i++) {
|
||||
if (is_file($stat)) {
|
||||
$j = json_decode((string)@file_get_contents($stat), true);
|
||||
if (is_array($j) && ($j['status'] ?? '') === 'running' && time() - filemtime($stat) < 60) {
|
||||
echo json_encode(['ok' => true, 'status' => 'running', 'action' => $_covAction]);
|
||||
exit;
|
||||
}
|
||||
}
|
||||
usleep(250000);
|
||||
}
|
||||
echo json_encode(['ok' => false,
|
||||
'error' => 'Job did not report as running — check the Fallback log. It refuses to run unless fallback state is NORMAL.']);
|
||||
exit;
|
||||
}
|
||||
if (($_POST['action'] ?? '') !== 'cover') {
|
||||
echo json_encode(['ok' => false, 'error' => 'Unknown action']);
|
||||
exit;
|
||||
|
||||
Reference in New Issue
Block a user