Gmer4Lfe
671e7ea5a4
fallback.sh starts covered containers with docker start and never creates them, so a coverage list the partner has never been sent is a promise nothing can keep — all twelve were missing. Adds the push and remove paths, a readiness card that checks rather than infers, and the fallback state the assistant needs to answer for it.
419 lines
21 KiB
PHP
419 lines
21 KiB
PHP
<?php
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
// PURPOSE
|
|
// Failover coverage: FALLBACK_<me>_TIER1-4 in this host's own conf — which of THIS host's
|
|
// containers the partner starts when this host goes dark, and in which delay tier.
|
|
//
|
|
// OPERATIONAL MODEL
|
|
// GET this host's containers plus current tier membership.
|
|
// POST action=cover tiers=<json {container: tier}> rewrites all four tier arrays.
|
|
//
|
|
// Originally the Coverage picker on the Partnership page (e8ee5b0), removed the same day in
|
|
// 1a836da — "they describe what the partner runs during an outage, which is the Fallback tab's
|
|
// subject" — and never rehomed, because that tab had nothing to host it. This is that card,
|
|
// rebuilt where it belongs, with the services half left behind: pushing XML templates to a
|
|
// mirror is an onboard concern, not a failover one.
|
|
//
|
|
// DESIGN PRINCIPLES
|
|
// Edit the array fallback.sh actually reads, not a parallel one.
|
|
// The coverage tiers already exist and already carry the timing. A second "what to fail
|
|
// over" list would be a second answer to the same question, and the two would drift.
|
|
//
|
|
// A host edits only its OWN tiers, and the page says so.
|
|
// FALLBACK_<host>_TIER* lives in that host's conf and describes what someone else runs for
|
|
// it. Sparse checkout means this host does not have the partner's host*.conf at all — only
|
|
// the read-only RAM cache conf_sync fills — so an editor for the partner's coverage would
|
|
// be writing to a cache that the next sync overwrites. Configure HOST2's coverage from
|
|
// HOST2. This is the same trap that left the Watchdog card reporting a partner's lists as
|
|
// empty when they were merely somewhere else.
|
|
//
|
|
// OPERATIONAL SAFEGUARDS
|
|
// POST only for writes, so Unraid's CSRF guard applies.
|
|
//
|
|
// Names are validated against containers this host runs, PLUS whatever the tiers already name.
|
|
// The conf legitimately holds entries for containers not present right now — removed,
|
|
// stopped, or renamed. Validating only against the running set would refuse to save a list
|
|
// the operator never touched. New names still have to be real; the guard is against
|
|
// inventing containers, not against keeping ones already recorded.
|
|
//
|
|
// A tier outside 1-4 is rejected, never clamped. Silently moving a container from tier 9 to
|
|
// tier 4 would give it a 24-hour delay nobody asked for.
|
|
//
|
|
// An absent array is refused, not appended. Writing a new block into an unknown position in a
|
|
// conf is how a setting ends up in the wrong section and stops being read.
|
|
//
|
|
// Writing an empty list is allowed — "cover nothing" is a legitimate choice and the only way
|
|
// to express it.
|
|
//
|
|
// REQUEST
|
|
// GET → current lists
|
|
// POST action=cover tiers={"Emby":1,...} → rewrite tiers 1-4
|
|
//
|
|
// RESPONSE
|
|
// {"ok":true,...} read payload, or {"ok":true,"counts":{...}} after a write
|
|
// {"ok":false,"error":string} validation or write failure, stated
|
|
//
|
|
// DEPENDS ON
|
|
// include/confform.php vv_conf_edit(), vv_conf_last_error(), vv_parse_conf_list()
|
|
// include/common.php vv_docker_containers()
|
|
// include/config.php vv_detect_host(), vv_read_conf_raw(), vv_push_master_conf()
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
header('Content-Type: application/json');
|
|
require_once dirname(__DIR__) . '/include/fallback.php'; // vv_fb_proc()
|
|
require_once dirname(__DIR__) . '/include/confform.php';
|
|
require_once dirname(__DIR__) . '/include/common.php';
|
|
|
|
$hostId = vv_detect_host();
|
|
$hostUp = strtoupper($hostId);
|
|
$myConf = $hostId . '.conf';
|
|
|
|
$TIERS = [1, 2, 3, 4];
|
|
$tierVar = fn(int $t) => "FALLBACK_{$hostUp}_TIER{$t}";
|
|
|
|
// ── Read ─────────────────────────────────────────────────────────────────────────────────────
|
|
if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
|
$raw = vv_read_conf_raw($myConf);
|
|
$mRaw = vv_read_conf_raw('master.conf');
|
|
|
|
$cover = [];
|
|
foreach ($TIERS as $t) {
|
|
// host*.conf first, master.conf second — installs that kept the tiers there still read.
|
|
$vals = vv_parse_conf_list($raw, $tierVar($t)) ?: vv_parse_conf_list($mRaw, $tierVar($t));
|
|
foreach ($vals as $c) { $c = trim($c); if ($c !== '') $cover[$c] = $t; }
|
|
}
|
|
|
|
$containers = [];
|
|
foreach (vv_docker_containers() as $c) {
|
|
$n = is_array($c) ? ($c['name'] ?? '') : (string)$c;
|
|
if ($n !== '') $containers[] = $n;
|
|
}
|
|
sort($containers, SORT_NATURAL | SORT_FLAG_CASE);
|
|
|
|
// Stacks are declared by the PARTNERSHIP OWNER and deployed to everyone, so on a mirror they
|
|
// are not in this host's conf at all — HOST2_PARTNERSHIP_AUTH_STACK is the shipped template,
|
|
// still commented out, while the eight auth containers it describes run there permanently
|
|
// because the owner put them there. Reading "this host's" stacks left the mirror's card
|
|
// showing every one of them as ordinary, selectable, uncovered.
|
|
//
|
|
// Owner's conf first, then this host's, unioned: on the owner the two are the same file, and
|
|
// a host that declares extras of its own still has them honoured. The owner's copy reaches a
|
|
// mirror through the conf_sync RAM cache, which vv_read_host_conf_raw() knows how to find.
|
|
$ownerSlot = strtolower(vv_parse_conf_scalar(vv_read_conf_raw('master.conf'), 'PARTNERSHIP_OWNER_HOST'));
|
|
$stackSrc = [];
|
|
foreach (array_unique(array_filter([$ownerSlot, $hostId])) as $slot) {
|
|
$stackSrc[strtoupper($slot)] = vv_read_host_conf_raw($slot);
|
|
}
|
|
|
|
$stackOf = [];
|
|
$byLower = [];
|
|
foreach ($containers as $n) $byLower[strtolower($n)] = $n;
|
|
foreach ($stackSrc as $id => $srcRaw) {
|
|
foreach ([
|
|
'auth' => "{$id}_PARTNERSHIP_AUTH_STACK",
|
|
'arrs' => "{$id}_PARTNERSHIP_ARR_STACK",
|
|
'services' => "{$id}_PARTNERSHIP_SERVICES_STACK",
|
|
] as $label => $var) {
|
|
foreach (vv_parse_conf_list($srcRaw, $var) as $xml) {
|
|
$n = preg_replace('/^my-|\.xml$/', '', trim($xml));
|
|
if ($n === '') continue;
|
|
// Only what this host actually runs. The owner's stack lists everything it
|
|
// deploys mesh-wide; a name with no container here is not "always up" here.
|
|
if (!isset($byLower[strtolower($n)])) continue;
|
|
$stackOf[$byLower[strtolower($n)]] = $label;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Named in a tier but not installed here. Reported rather than filtered: a tier entry for a
|
|
// container that does not exist is a line fallback.sh fails on during an outage, which is
|
|
// the worst possible moment to find a typo.
|
|
$missing = [];
|
|
$have = array_map('strtolower', $containers);
|
|
foreach (array_keys($cover) as $n) if (!in_array(strtolower($n), $have, true)) $missing[] = $n;
|
|
|
|
echo json_encode([
|
|
'ok' => true,
|
|
'host' => $hostUp,
|
|
'containers' => $containers,
|
|
'cover' => (object)$cover,
|
|
'stacks' => (object)$stackOf,
|
|
'missing' => $missing,
|
|
'tier_vars' => array_map($tierVar, $TIERS),
|
|
]);
|
|
exit;
|
|
}
|
|
|
|
// ── Write ────────────────────────────────────────────────────────────────────────────────────
|
|
// ── Readiness: would a failover actually work right now ─────────────────────────────────────
|
|
// Every row is a deterministic check with a stated basis. The assistant on this page EXPLAINS
|
|
// these rows; it never produces them. A model must not be the thing that says failover is ready —
|
|
// that is precisely the class of answer this codebase keeps finding to be confidently wrong, and
|
|
// on 2026-08-23 the coverage card itself was the confidently wrong surface: 12 containers listed,
|
|
// none of them present on the partner.
|
|
//
|
|
// Rows are ordered by what breaks first, not by severity, so reading top to bottom follows the
|
|
// order a real outage would hit them.
|
|
if (($_POST['action'] ?? '') === 'readiness') {
|
|
$rows = [];
|
|
$add = function (string $id, string $label, string $verdict, string $detail, string $ask = '')
|
|
use (&$rows) {
|
|
// verdict: ok | warn | fail | unknown — unknown is never dressed up as ok
|
|
$rows[] = ['id' => $id, 'label' => $label, 'verdict' => $verdict,
|
|
'detail' => $detail, 'ask' => $ask];
|
|
};
|
|
|
|
$me = vv_detect_host();
|
|
$ME = strtoupper($me);
|
|
$conf = vv_read_conf_raw('master.conf');
|
|
$hc = vv_read_conf_raw($me . '.conf');
|
|
|
|
// 1. is fallback even armed
|
|
$fbEnabled = preg_match('/^\s*FALLBACK_ENABLED\s*=\s*"?(\w+)/m', $conf, $m) ? $m[1] : 'unset';
|
|
$add('enabled', 'Fallback armed',
|
|
$fbEnabled === 'true' ? 'ok' : 'fail',
|
|
'FALLBACK_ENABLED=' . $fbEnabled,
|
|
'FALLBACK_ENABLED is ' . $fbEnabled . ' — what does that mean for a real outage?');
|
|
|
|
// 2. current state — anything but NORMAL means it is already doing something
|
|
$stateFile = STATE_DIR . '/fallback_state.db';
|
|
$state = 'unknown';
|
|
if (is_readable($stateFile) && preg_match('/^state=(\S+)/m', (string)@file_get_contents($stateFile), $m)) {
|
|
$state = $m[1];
|
|
}
|
|
// Same rule the node cards use: no state file plus a live daemon means the node has simply
|
|
// never transitioned, which is health, not ignorance. Reading the file alone gives a healthy
|
|
// node the same verdict as one whose daemon is dead.
|
|
$daemon = function_exists('vv_fb_proc') ? (vv_fb_proc('fallback')['running'] ?? false) : false;
|
|
$inferred = false;
|
|
if ($state === 'unknown' && $daemon) { $state = 'NORMAL'; $inferred = true; }
|
|
|
|
$add('state', 'State is NORMAL',
|
|
$state === 'NORMAL' ? 'ok' : ($state === 'unknown' ? 'unknown' : 'warn'),
|
|
'state=' . $state . ($inferred ? ' (from the live daemon — never transitioned)' : ''),
|
|
'Fallback state is ' . $state . '. What does that mean and what should I check?');
|
|
|
|
// 3. coverage configured at all
|
|
$covered = [];
|
|
for ($t = 1; $t <= 4; $t++) {
|
|
foreach (vv_parse_conf_list($hc, "FALLBACK_{$ME}_TIER{$t}") as $c) $covered[] = $c;
|
|
}
|
|
$add('coverage', 'Containers are covered',
|
|
$covered ? 'ok' : 'fail',
|
|
$covered ? count($covered) . ' container(s) across the tiers' : 'no containers in any tier',
|
|
$covered
|
|
? 'Walk me through what happens if this host goes dark right now, tier by tier, with the delays.'
|
|
: 'Nothing is listed in my fallback tiers — what would happen if this host went dark?');
|
|
|
|
// 4. THE one that was silently false — does the partner actually hold them
|
|
$cache = (defined('VV_CACHE_ROOT') ? VV_CACHE_ROOT : '/tmp/varaverk') . '/api/fallback_presence.json';
|
|
if (!is_readable($cache)) {
|
|
$add('present', 'Partner has the containers', 'unknown',
|
|
'never checked — run the presence check',
|
|
'How do I find out whether the partner actually has my covered containers?');
|
|
} else {
|
|
$j = json_decode((string)@file_get_contents($cache), true);
|
|
$miss = (array)($j['missing'] ?? []);
|
|
$age = time() - (int)@filemtime($cache);
|
|
$when = $age < 3600 ? round($age / 60) . 'm ago' : round($age / 3600) . 'h ago';
|
|
$add('present', 'Partner has the containers',
|
|
$miss ? 'fail' : 'ok',
|
|
$miss ? count($miss) . ' of ' . count($covered) . ' missing (' . $when . '): '
|
|
. implode(', ', array_slice($miss, 0, 4)) . (count($miss) > 4 ? '…' : '')
|
|
: 'all ' . count($covered) . ' present (' . $when . ')',
|
|
$miss ? 'The partner is missing ' . implode(', ', array_slice($miss, 0, 6))
|
|
. '. What happens during a failover, and how do I fix it?' : '');
|
|
}
|
|
|
|
// 5. host-specific networks that cannot be recreated on the partner
|
|
$wg = [];
|
|
foreach ($covered as $c) {
|
|
foreach (glob('/boot/config/plugins/dockerMan/templates-user/*.xml') as $x) {
|
|
$t = @file_get_contents($x);
|
|
if ($t === false || strpos($t, "<Name>$c</Name>") === false) continue;
|
|
if (preg_match('~<Network>(wg\d+)</Network>~', $t, $m)) $wg[] = "$c ({$m[1]})";
|
|
break;
|
|
}
|
|
}
|
|
if ($wg) {
|
|
$add('wgnet', 'No tunnel-bound networks', 'warn',
|
|
implode(', ', $wg),
|
|
'Some covered containers use a WireGuard-backed network. Why can that not move to the partner?');
|
|
}
|
|
|
|
// 6. handback writeback — invisible until the day it matters
|
|
$wb = preg_match('/^\s*FALLBACK_RSYNC_ENABLED\s*=\s*"?(\w+)/m', $conf, $m) ? $m[1] : 'unset';
|
|
$add('writeback', 'Handback writeback', $wb === 'true' ? 'ok' : 'warn',
|
|
'FALLBACK_RSYNC_ENABLED=' . $wb,
|
|
'FALLBACK_RSYNC_ENABLED is ' . $wb . ' — what do I lose on handback?');
|
|
|
|
// Overall verdict is the worst row, never an average. One failed check is a failed failover.
|
|
$order = ['ok' => 0, 'warn' => 1, 'unknown' => 2, 'fail' => 3];
|
|
$worst = 'ok';
|
|
foreach ($rows as $r) if ($order[$r['verdict']] > $order[$worst]) $worst = $r['verdict'];
|
|
|
|
echo json_encode(['ok' => true, 'verdict' => $worst, 'rows' => $rows,
|
|
'summary' => $worst === 'ok'
|
|
? 'Every check passed'
|
|
: ($worst === 'fail' ? 'A failover would NOT work as configured'
|
|
: 'Failover is configured but something needs a look')]);
|
|
exit;
|
|
}
|
|
// ── Push / remove / status: what the PARTNER actually holds ──────────────────────────────────
|
|
// Coverage names a container; fallback.sh starts it with `docker start`, which fails unless the
|
|
// partner already has it built. Measured 2026-08-23: 12 of 12 covered containers were absent from
|
|
// the partner, so every tier would have failed on the first real outage. These three actions are
|
|
// how the card closes and inspects that gap.
|
|
//
|
|
// Deliberately NOT folded into `cover`. Saving a tier list is a cheap, reversible config write;
|
|
// deploying a dozen containers onto another machine is neither, and a stray click should not be
|
|
// able to do it.
|
|
$_covAction = $_POST['action'] ?? '';
|
|
if (in_array($_covAction, ['push', 'remove', 'deploy_status'], true)) {
|
|
$dir = rtrim(SCRIPTS_DIR, '/');
|
|
$script = $dir . '/Fallback/coverage_deploy.sh';
|
|
$runner = $dir . '/Plugin/unraid/run_job.sh';
|
|
|
|
if (!is_file($script)) {
|
|
echo json_encode(['ok' => false, 'error' => 'coverage_deploy.sh not found on this host']);
|
|
exit;
|
|
}
|
|
|
|
// Status is read-only and fast enough to answer inline; the two that change the partner are
|
|
// dispatched to run_job.sh so they get a job record, a log, and a UI surface like every other
|
|
// long operation here.
|
|
if ($_covAction === 'deploy_status') {
|
|
$out = [];
|
|
exec('timeout 120 /bin/bash ' . escapeshellarg($script) . ' --status 2>&1', $out, $rc);
|
|
$present = []; $missing = [];
|
|
foreach ($out as $line) {
|
|
if (preg_match('/^\s{2}(\S+)\s+MISSING on/', $line, $m)) $missing[] = $m[1];
|
|
elseif (preg_match('/^\s{2}(\S+)\s+on \S+ \((\w+)\)/', $line, $m)) $present[$m[1]] = $m[2];
|
|
}
|
|
echo json_encode([
|
|
'ok' => true,
|
|
'present' => $present,
|
|
'missing' => $missing,
|
|
// rc 2 means "ran fine, some are missing" — not a failure of the check itself.
|
|
'checked' => ($rc === 0 || $rc === 2),
|
|
]);
|
|
exit;
|
|
}
|
|
|
|
if (!is_file($runner)) {
|
|
echo json_encode(['ok' => false, 'error' => 'run_job.sh not found on this host']);
|
|
exit;
|
|
}
|
|
$flag = $_covAction === 'push' ? '--push' : '--remove';
|
|
$stat = '/var/log/varaverk/Fallback/coverage_deploy.json';
|
|
|
|
shell_exec('setsid /bin/bash ' . escapeshellarg($runner)
|
|
. ' ' . escapeshellarg('Fallback/coverage_deploy.sh')
|
|
. ' ' . escapeshellarg($script)
|
|
. ' --manual ' . escapeshellarg($flag)
|
|
. ' >/dev/null 2>&1 </dev/null &');
|
|
|
|
// Report what the record says, not that the command was issued — run_job.sh writes its stat
|
|
// file before running, so a live record is the difference between a job that started and one
|
|
// refused for already running, or killed by the NORMAL-state gate.
|
|
for ($i = 0; $i < 12; $i++) {
|
|
if (is_file($stat)) {
|
|
$j = json_decode((string)@file_get_contents($stat), true);
|
|
if (is_array($j) && ($j['status'] ?? '') === 'running' && time() - filemtime($stat) < 60) {
|
|
echo json_encode(['ok' => true, 'status' => 'running', 'action' => $_covAction]);
|
|
exit;
|
|
}
|
|
}
|
|
usleep(250000);
|
|
}
|
|
echo json_encode(['ok' => false,
|
|
'error' => 'Job did not report as running — check the Fallback log. It refuses to run unless fallback state is NORMAL.']);
|
|
exit;
|
|
}
|
|
if (($_POST['action'] ?? '') !== 'cover') {
|
|
echo json_encode(['ok' => false, 'error' => 'Unknown action']);
|
|
exit;
|
|
}
|
|
|
|
$known = [];
|
|
foreach (vv_docker_containers() as $c) {
|
|
$n = is_array($c) ? ($c['name'] ?? '') : (string)$c;
|
|
if ($n !== '') $known[strtolower($n)] = $n;
|
|
}
|
|
$existingRaw = vv_read_conf_raw($myConf);
|
|
foreach ($TIERS as $t) {
|
|
foreach (vv_parse_conf_list($existingRaw, $tierVar($t)) as $n) {
|
|
$n = trim($n);
|
|
if ($n !== '' && !isset($known[strtolower($n)])) $known[strtolower($n)] = $n;
|
|
}
|
|
}
|
|
|
|
|
|
// Stack containers refused here, not only greyed out in the picker. A disabled select is a
|
|
// courtesy to the operator, not a constraint on the endpoint.
|
|
//
|
|
// Sourced from the OWNER's conf as well as this host's, for the same reason the read path is: on
|
|
// a mirror the stack it runs is the owner's declaration, and checking only the local conf would
|
|
// have let a mirror assign a fallback tier to a container that never stops.
|
|
$ownerSlotW = strtolower(vv_parse_conf_scalar(vv_read_conf_raw('master.conf'), 'PARTNERSHIP_OWNER_HOST'));
|
|
$stackNames = [];
|
|
foreach (array_unique(array_filter([$ownerSlotW, $hostId])) as $slot) {
|
|
$srcRaw = vv_read_host_conf_raw($slot);
|
|
$id = strtoupper($slot);
|
|
foreach ([
|
|
"{$id}_PARTNERSHIP_AUTH_STACK",
|
|
"{$id}_PARTNERSHIP_ARR_STACK",
|
|
"{$id}_PARTNERSHIP_SERVICES_STACK",
|
|
] as $var) {
|
|
foreach (vv_parse_conf_list($srcRaw, $var) as $xml) {
|
|
$n = preg_replace('/^my-|\.xml$/', '', trim($xml));
|
|
if ($n !== '') $stackNames[strtolower($n)] = true;
|
|
}
|
|
}
|
|
}
|
|
$map = json_decode((string)($_POST['tiers'] ?? ''), true);
|
|
if (!is_array($map)) { echo json_encode(['ok' => false, 'error' => 'tiers must be an object']); exit; }
|
|
|
|
$byTier = array_fill_keys($TIERS, []);
|
|
foreach ($map as $name => $tier) {
|
|
$t = (int)$tier;
|
|
if (!in_array($t, $TIERS, true)) {
|
|
echo json_encode(['ok' => false, 'error' => "Tier $tier is not 1-4 (for $name)"]); exit;
|
|
}
|
|
if (!isset($known[strtolower((string)$name)])) {
|
|
echo json_encode(['ok' => false, 'error' => "No container named $name on this host"]); exit;
|
|
}
|
|
if (isset($stackNames[strtolower((string)$name)])) {
|
|
// Direction-neutral wording: on the owner this container is deployed TO the partner, on a
|
|
// mirror it was deployed HERE by the owner. Both mean the same thing for coverage — it
|
|
// runs on both nodes continuously, so there is nothing for a tier to start.
|
|
echo json_encode(['ok' => false,
|
|
'error' => "$name belongs to a partnership stack — it runs on both nodes continuously, so it cannot be given a fallback tier"]); exit;
|
|
}
|
|
$byTier[$t][] = $known[strtolower((string)$name)];
|
|
}
|
|
|
|
// Rewrites one `NAME=(` … `)` block in place, preserving the conf's leading indent.
|
|
$rewrite = function (string $cur, string $var, array $items): ?string {
|
|
$body = '';
|
|
foreach ($items as $i) $body .= " \"" . $i . "\"\n";
|
|
$pattern = '/^([ \t]*)' . preg_quote($var, '/') . '=\((?:[^)]*)\)/m';
|
|
if (!preg_match($pattern, $cur)) return null; // absent: refuse rather than append blind
|
|
return preg_replace_callback($pattern,
|
|
fn($m) => $m[1] . $var . "=(\n" . $body . $m[1] . ")", $cur, 1);
|
|
};
|
|
|
|
$ok = vv_conf_edit($myConf, function (string $cur) use ($byTier, $TIERS, $tierVar, $rewrite): ?string {
|
|
foreach ($TIERS as $t) {
|
|
$next = $rewrite($cur, $tierVar($t), $byTier[$t]);
|
|
if ($next === null) return null;
|
|
$cur = $next;
|
|
}
|
|
return $cur;
|
|
}, [], array_map($tierVar, $TIERS));
|
|
|
|
if (!$ok) { echo json_encode(['ok' => false, 'error' => vv_conf_last_error() ?: 'Write failed']); exit; }
|
|
|
|
// A partner holding the old list is a partner that will act on the old list.
|
|
vv_push_master_conf();
|
|
echo json_encode(['ok' => true, 'counts' => array_map('count', $byTier)]);
|