FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild the Fallback tab around what the daemon is actually doing, not what its last state file said

Browse Source
This commit is contained in:
Gmer4Lfe
2026-08-22 00:44:35 -04:00
parent 5924aae1c0
commit da15cc7531
3 changed files with 518 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Plugin/unraid/pages/fallback.php
+296 -36
View File
@@ -16,22 +16,81 @@
// outage, when stale numbers are actively misleading.
//
// OPERATIONAL SAFEGUARDS
// Read-only. The page cannot trigger a failover, force a handback, or start a covered
// container. Fallback is driven by fallback.sh reacting to real reachability, and a manual
// override from a browser is exactly the wrong way to enter that state.
// Cannot cause a failover. The page stops daemons, starts a PREVIEW, and edits conf — it has
// no control that enters FALLBACK or forces a handback. Fallback is driven by fallback.sh
// reacting to real reachability, and a manual override from a browser is exactly the wrong
// way to enter that state. The one start control is --dry-run, which changes nothing.
//
// A missing state file renders as unknown, never as NORMAL — claiming healthy for a
// fallback process that is not running would be the worst possible error on this page.
//
// State age is shown next to state, always. Every value in the state file was written by a
// daemon that may not be running: this host displayed NORMAL from a file five days stale
// with no process alive, and nothing on the page said so.
//
// Stopping the test is a different button from stopping the daemon, deliberately. The test
// holds an iptables rule that only its own EXIT trap removes, so the two cannot share a
// control that might escalate to SIGKILL.
//
// RENDERS
// Per-node state, tier activation and delays, handback strikes, covered container status
// Per-node state and freshness, daemon liveness and mode, reachability legs, tier activation
// and delays, handback strikes, covered container status, quick settings, assistant
//
// DEPENDS ON
// api/fallback.php polled every 30s → include/fallback.php
// api/confform.php inline conf edits → include/confform.php
// api/fallback.php polled every 30s → include/fallback.php
// api/fallback_control.php start/stop actions → Fallback/fallback*.sh
// api/confform.php inline conf edits → include/confform.php
require_once dirname(__DIR__) . '/include/confui.php';
require_once dirname(__DIR__) . '/include/ai_chat.php';
// Same shared surface as the Partnership and Monitor tabs, gated the same way. Fallback
// questions — why a tier has not fired, what the mirror would actually start, whether a stale
// state file matters — are asked while looking at this page.
if (vv_ai_ui_on()) vv_ai_chat_assets();
?>
<style>
/* ── Host card: daemon, freshness, reachability, controls ── */
/* Class names are all vv-fb-* prefixed. Unraid Connect injects a global Tailwind layer into
every page, so a bare utility-shaped name like `fixed` or `grid` would be captured by it. */
.vv-fb-hcard { background:#141414;border:1px solid #262626;border-radius:6px;padding:12px 13px;min-width:0; }
.vv-fb-hcard.me { border-color:#2a3a2a; }
.vv-fb-hcard.warn { border-color:#4a3800; }
.vv-fb-hrow { display:flex;align-items:center;gap:7px;margin-bottom:8px;flex-wrap:wrap; }
.vv-fb-hid { font-size:13px;font-weight:700;color:#ddd; }
.vv-fb-hnm { font-size:11px;color:#666;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;min-width:0; }
.vv-fb-usbadge { background:#1a3a1a;color:#4caf50;font-size:8px;padding:1px 5px;border-radius:3px; }
.vv-fb-stats { display:grid;grid-template-columns:auto 1fr;gap:3px 10px;font-size:11px;margin-bottom:9px; }
.vv-fb-stats b { color:#444;font-weight:400; }
.vv-fb-sv { color:#999;min-width:0; }
.vv-fb-sv.good { color:#4caf50; }
.vv-fb-sv.warn { color:#ffb74d; }
.vv-fb-sv.bad { color:#ef5350; }
.vv-fb-sv.dim { color:#3a3a3a; }
/* Reachability legs — separate facts, never rolled into one boolean. */
.vv-fb-legs { display:flex;gap:5px;flex-wrap:wrap;margin-bottom:9px; }
.vv-fb-leg { font-size:9px;padding:1px 6px;border-radius:3px;border:1px solid #222;background:#111;color:#444; }
.vv-fb-leg.ok { border-color:#243a24;background:#101a10;color:#4caf50; }
.vv-fb-leg.no { border-color:#3a1e1e;background:#1a1010;color:#ef5350; }
.vv-fb-acts { display:flex;gap:6px;flex-wrap:wrap;margin-top:9px;padding-top:8px;border-top:1px solid #1e1e1e; }
.vv-fb-btn { background:#111;border:1px solid #262626;color:#888;border-radius:3px;
padding:4px 9px;cursor:pointer;font-size:10px;white-space:nowrap; }
.vv-fb-btn:hover:not(:disabled) { border-color:#3a3a3a;color:#ccc; }
.vv-fb-btn:disabled { opacity:.4;cursor:default; }
.vv-fb-btn.stop { color:#ef9a9a; }
.vv-fb-btn.go { color:#4a9eff; }
.vv-fb-out { font-size:9px;color:#555;margin-top:6px;white-space:pre-wrap;word-break:break-word;
max-height:80px;overflow:auto;font-family:monospace; }
/* Quick settings — one card per setting rather than one card of rows, so each carries its own
why. The old single card listed three numbers with no indication what changing one costs. */
.vv-fb-qs { display:grid;grid-template-columns:repeat(auto-fit,minmax(190px,1fr));gap:10px;margin-bottom:12px; }
.vv-fb-qcard { background:#141414;border:1px solid #242424;border-radius:6px;padding:10px 11px; }
.vv-fb-qlbl { font-size:11px;color:#aaa;font-weight:600;margin-bottom:2px; }
.vv-fb-qsub { font-size:9px;color:#3f3f3f;line-height:1.35;margin-bottom:7px; }
.vv-fb-qin { display:flex;align-items:center;gap:7px; }
/* ── Existing status styles ── */
.vv-fb-active { background:#1a1200;border:1px solid #5a3800;border-radius:6px;padding:12px 14px; }
.vv-fb-active-h { display:flex;align-items:baseline;gap:10px;margin-bottom:8px; }
@@ -125,50 +184,77 @@ require_once dirname(__DIR__) . '/include/confui.php';
</div>
</div>
<!-- Mesh verdict — one line answering "is this armed and is anything watching" before any detail -->
<div class="vv-fb-card" id="vv-fb-verdict-card" style="margin-bottom:12px;">
<div id="vv-fb-verdict" style="font-size:12px;color:#555;">Loading…</div>
</div>
<!-- Status section -->
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;padding:0 2px;">
<span style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;">Status</span>
<span style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;">Nodes</span>
<span style="font-size:11px;color:#3a3a3a;" id="vv-fb-ts"></span>
</div>
<div id="vv-fb-grid" style="display:grid;grid-template-columns:repeat(8,1fr);gap:12px;margin-bottom:14px;">
<div id="vv-fb-grid" style="display:grid;grid-template-columns:repeat(auto-fit,minmax(340px,1fr));gap:12px;margin-bottom:14px;">
<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">Loading…</div>
</div>
<!-- Settings card -->
<div class="vv-fb-card">
<div class="vv-fb-card-hdr">Settings</div>
<!-- Assistant + mesh chat -->
<?php if (vv_ai_ui_on()): ?>
<div class="vv-card" id="vv-fb-ai-card" style="margin-bottom:12px;">
<?php vv_ai_chat_markup('vv-fb-ai', [
'profile' => 'varaverk',
'compact' => true,
// Fallback is the mesh's business by definition — a failover is two hosts agreeing about
// each other — so the mesh side leads here, as it does on Partnership.
'mesh' => true,
'meshDefault' => true,
'title' => 'Assistant',
'height' => '300px',
'tall' => '500px',
'tallLarge' => '750px',
'empty' => 'Ask about fallback — why a tier has not fired, what the partner would '
. 'start if this host went dark, whether a stale state file matters.',
]); ?>
</div>
<?php endif; ?>
<div class="vv-fb-set-row">
<span class="vv-fb-set-lbl">Check interval</span>
<div style="display:flex;align-items:center;gap:8px;">
<!-- Quick settings -->
<div style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;margin-bottom:10px;padding:0 2px;">Quick settings</div>
<div class="vv-fb-qs">
<div class="vv-fb-qcard">
<div class="vv-fb-qlbl">Check interval</div>
<div class="vv-fb-qsub">How often the daemon re-tests the partner. Also the age at which a state file is stale.</div>
<div class="vv-fb-qin">
<input class="vv-fb-set-inp" id="vv-fb-interval" type="number" min="5" max="300">
<span class="vv-fb-set-unit">seconds</span>
</div>
</div>
<div class="vv-fb-set-row">
<span class="vv-fb-set-lbl">Handback strikes</span>
<div style="display:flex;align-items:center;gap:8px;">
<div class="vv-fb-qcard">
<div class="vv-fb-qlbl">Handback strikes</div>
<div class="vv-fb-qsub">Consecutive healthy checks before handing services back. Higher rides out a flapping link.</div>
<div class="vv-fb-qin">
<input class="vv-fb-set-inp" id="vv-fb-strikes" type="number" min="1" max="20">
<span class="vv-fb-set-unit">consecutive</span>
</div>
</div>
<div class="vv-fb-set-row">
<span class="vv-fb-set-lbl">Partnership suspend after</span>
<div style="display:flex;align-items:center;gap:8px;">
<div class="vv-fb-qcard">
<div class="vv-fb-qlbl">Partnership suspend after</div>
<div class="vv-fb-qsub">Minutes without an active partnership before fallback suspends itself. 0 suspends immediately.</div>
<div class="vv-fb-qin">
<input class="vv-fb-set-inp" id="vv-fb-suspend" type="number" min="0" max="1440">
<span class="vv-fb-set-unit">minutes</span>
</div>
</div>
<div style="display:flex;justify-content:flex-end;align-items:center;gap:10px;margin-top:10px;padding-top:8px;border-top:1px solid #1e1e1e;">
<span id="vv-fb-set-fb" style="font-size:11px;"></span>
<button class="vv-fb-save-btn" id="vv-fb-save-btn" onclick="vvFbSaveSettings()">Save</button>
</div>
</div>
<div style="display:flex;justify-content:flex-end;align-items:center;gap:10px;margin-bottom:14px;">
<span id="vv-fb-set-fb" style="font-size:11px;"></span>
<button class="vv-fb-save-btn" id="vv-fb-save-btn" onclick="vvFbSaveSettings()">Save</button>
</div>
<?php
// The conf sections this page is about, drawn by the shared renderer. They were reachable
@@ -303,6 +389,9 @@ function _tierSection(tiers, activeTier, delays) {
}).join('');
}
// Only rendered against FRESH state — see _nodeCard. A grace timer counted off a state file
// nobody has written for five days reads as an active grace window, which is precisely the kind
// of stale-as-current claim this page must not make.
function _ptStatus(st) {
if (!st) return '';
if (st.partnership_suspended) return _stateBadge('SUSPENDED');
@@ -313,37 +402,153 @@ function _ptStatus(st) {
return '';
}
function _nodeCard(node, suspendAfter) {
function _age(sec) {
if (sec === null || sec === undefined) return '—';
if (sec < 60) return sec + 's ago';
if (sec < 3600) return Math.floor(sec / 60) + 'm ago';
if (sec < 86400) return Math.floor(sec / 3600) + 'h ago';
return Math.floor(sec / 86400) + 'd ago';
}
// Daemon liveness, stated as one of four distinct things rather than a boolean. "Not running"
// and "running a preview" are opposite operational facts and the old page showed neither.
function _daemonRow(p, checkInterval) {
if (!p) return ['dim', 'unknown', ''];
if (p.stale_lock) return ['bad', 'not running · stale lock' + (p.pid ? ' (PID ' + p.pid + ')' : ''), 'stale'];
if (p.running === null) return ['dim', 'unknown — partner not reachable', ''];
if (!p.running) return ['bad', 'not running', ''];
const mode = p.mode === 'dry-run'
? '<span style="color:#4a9eff;">dry run</span>'
: '<span style="color:#4caf50;">live</span>';
return ['good', mode + ' · PID ' + p.pid, 'up'];
}
// State freshness. The daemon rewrites this file every check interval, so anything past a few
// intervals is a wedged process or a dead one — and the state value it left behind is a claim
// about the past, not the present.
function _freshness(node, checkInterval) {
const a = node.state_age;
if (a === null || a === undefined) {
return node.is_me
? ['bad', 'no state file — fallback has never run here']
: ['dim', 'not readable from here'];
}
const limit = Math.max(120, (checkInterval || 30) * 4);
return [a > limit ? 'bad' : 'good', _age(a) + (a > limit ? ' — stale' : '')];
}
function _leg(ok, label) {
const cls = ok === true ? 'ok' : ok === false ? 'no' : '';
const sym = ok === true ? '✓' : ok === false ? '✕' : '·';
return `<span class="vv-fb-leg ${cls}">${sym} ${label}</span>`;
}
function _nodeCard(node, data) {
const st = node.state || {};
const state = st.partnership_suspended ? 'SUSPENDED' : (st.state || 'UNKNOWN');
const cov = node.covers;
const active = _activeTier(state === 'FALLBACK' ? st : null);
const slot = node.slot;
const [dCls, dTxt, dKind] = _daemonRow(node.proc, data.check_interval);
const [fCls, fTxt] = _freshness(node, data.check_interval);
const test = node.proc_test || {};
const reach = node.reach || {};
const covTarget = cov
? `<span class="vv-fb-arrow">→</span><span class="vv-fb-covers">covers ${cov.id} (${cov.hostname})</span>`
: '';
? `<span class="vv-fb-sv">${cov.id} <span style="color:#3a3a3a;">(${vvEscHtml(cov.hostname)})</span></span>`
: '<span class="vv-fb-sv dim">nothing configured</span>';
const tierSection = cov
? _tierSection(cov, active, cov.delays)
: '<div style="color:#3a3a3a;font-size:11px;">No coverage configured</div>';
const ptBadge = node.is_me ? _ptStatus(st) : '';
// Tier lists live in the COVERED host's conf, so an empty set here is a config gap on the
// other side, not on this one. Saying which conf to edit saves the hunt.
const tierTotal = cov ? ['tier1','tier2','tier3','tier4'].reduce((n,k)=>n+((cov[k]||[]).length),0) : 0;
const tierWarn = (cov && tierTotal === 0)
? `<div style="font-size:9px;color:#5a4020;margin-top:5px;">No containers in any tier — set FALLBACK_${cov.id}_TIER1 in ${cov.slot}.conf</div>`
: '';
return `<div class="vv-card vv-fb-node">
<div class="vv-fb-node-h">
const busy = _vvFbBusy[slot] ? 'disabled' : '';
const acts = `
<button class="vv-fb-btn stop" ${busy} onclick="vvFbAct('${slot}','stop',this)"
title="fallback.sh --stop — SIGTERM, then SIGKILL after 10s">Stop fallback</button>
<button class="vv-fb-btn go" ${busy} onclick="vvFbAct('${slot}','start_dry',this)"
title="fallback.sh --dry-run — previews decisions, changes nothing">Start dry run</button>
<button class="vv-fb-btn stop" ${busy} onclick="vvFbAct('${slot}','stop_test',this)"
title="fallback_test.sh --stop — SIGTERM only, never SIGKILL: only its trap removes the iptables rule">Stop test</button>
${(node.proc?.stale_lock || test.stale_lock)
? `<button class="vv-fb-btn" ${busy} onclick="vvFbAct('${slot}','clear_lock',this)"
title="Remove lock files left by a killed run">Clear lock</button>` : ''}`;
const testRow = test.running
? `<b>Test</b><span class="vv-fb-sv warn">running · PID ${test.pid} — holds an iptables rule</span>`
: test.stale_lock
? `<b>Test</b><span class="vv-fb-sv bad">stale lock — an iptables rule may be stranded</span>`
: '';
return `<div class="vv-fb-hcard${node.is_me ? ' me' : ''}${(dCls === 'bad' || fCls === 'bad') ? ' warn' : ''}">
<div class="vv-fb-hrow">
${_stateDot(state)}
<span class="vv-fb-node-id">${node.id}</span>
<span class="vv-fb-node-nm">${node.hostname}</span>
${covTarget}
<span class="vv-fb-hid">${node.id}</span>
<span class="vv-fb-hnm">${vvEscHtml(node.hostname)}</span>
${node.is_me ? '<span class="vv-fb-usbadge">US</span>' : ''}
<span style="flex:1"></span>
${ptBadge}
${(node.is_me && fCls !== 'bad') ? _ptStatus(st) : ''}
${_stateBadge(state)}
</div>
<div class="vv-fb-legs">
${_leg(reach.tailscale, 'tailscale')}
${_leg(node.is_me ? true : reach.ssh, 'ssh')}
${_leg(reach.state_file, 'state file')}
${node.ts_ip ? `<span class="vv-fb-leg">${vvEscHtml(node.ts_ip)}</span>` : ''}
</div>
<div class="vv-fb-stats">
<b>Daemon</b> <span class="vv-fb-sv ${dCls}">${dTxt}</span>
<b>State age</b> <span class="vv-fb-sv ${fCls}">${fTxt}</span>
<b>Covers</b> ${covTarget}
<b>Running</b> <span class="vv-fb-sv">${node.running_count ?? 0} containers</span>
${state === 'FALLBACK'
? `<b>Strikes</b><span class="vv-fb-sv warn">${st.handback_strikes} / ${data.handback_req || 3}</span>` : ''}
${testRow}
</div>
<hr class="vv-fb-sep">
${tierSection}
${tierWarn}
<div class="vv-fb-acts">${acts}</div>
<div class="vv-fb-out" id="vv-fb-out-${slot}"></div>
</div>`;
}
// One mesh-level sentence, above the detail. Answers the two questions the page exists for
// before anyone has to read a card: is fallback armed, and is anything actually watching.
function _verdict(data) {
const el = document.getElementById('vv-fb-verdict');
if (!el) return;
const nodes = data.nodes || [];
if (!data.partnership_enabled)
return void (el.innerHTML = '<span style="color:#ef5350;">PARTNERSHIP_ENABLED=false</span> — all cross-server operations are disabled.');
if (!data.fb_enabled)
return void (el.innerHTML = '<span style="color:#ffb74d;">FALLBACK_ENABLED=false</span> — nothing will fail over. Arming it takes effect at the next array start.');
const inFb = nodes.filter(n => (n.state?.state) === 'FALLBACK');
const live = nodes.filter(n => n.proc?.running && n.proc.mode === 'live');
const dry = nodes.filter(n => n.proc?.running && n.proc.mode === 'dry-run');
const dead = nodes.filter(n => n.proc && n.proc.running === false);
if (inFb.length)
return void (el.innerHTML = `<span style="color:#ffb74d;font-weight:600;">${inFb.map(n=>n.id).join(', ')} in FALLBACK</span> — covering for a partner right now.`);
let s = `<span style="color:#4caf50;">Armed</span> · ${live.length}/${nodes.length} node${nodes.length!==1?'s':''} running the daemon`;
if (dry.length) s += ` · <span style="color:#4a9eff;">${dry.length} in dry run</span>`;
if (dead.length) s += ` · <span style="color:#ef5350;">${dead.map(n=>n.id).join(', ')} not running — nothing would detect an outage there</span>`;
el.innerHTML = s;
}
function _setToggles(data) {
const pairs = [
['vv-fb-pt-tog', !!data.partnership_enabled],
@@ -369,8 +574,10 @@ function _setInputs(data) {
}
function _render(data) {
_vvFbLast = data;
_setToggles(data);
_setInputs(data);
_verdict(data);
const grid = document.getElementById('vv-fb-grid');
@@ -381,7 +588,7 @@ function _render(data) {
} else {
const nodes = data.nodes || [];
let html = _activeCard(nodes, data.handback_req || 3);
for (const node of nodes) html += _nodeCard(node, data.suspend_after || 120);
for (const node of nodes) html += _nodeCard(node, data);
grid.innerHTML = html || '<div class="vv-fb-disabled">No nodes configured.</div>';
}
@@ -403,6 +610,59 @@ function vvFbLoad() {
});
}
// Per-slot in-flight guard, so a second click cannot fire while a stop is still escalating.
// Keyed by slot rather than one page-wide flag: acting on one host must not freeze the other's
// controls, and on this page the two hosts are routinely in different states.
const _vvFbBusy = {};
let _vvFbLast = null;
// URLSearchParams, not FormData. Unraid's CSRF token injector is jQuery-only, and a multipart
// body from native fetch() hangs here with no status and no server-side trace.
window.vvFbAct = async function (slot, action, btn) {
if (_vvFbBusy[slot]) return;
const labels = {
stop: 'Stop the fallback daemon on ' + slot.toUpperCase() + '?',
stop_test: 'Stop the fallback TEST on ' + slot.toUpperCase() + '?\n\nSIGTERM only — the test needs its own trap to remove the iptables rule it installed.',
start_dry: 'Start a fallback DRY RUN on ' + slot.toUpperCase() + '?\n\nIt previews decisions and changes nothing.',
clear_lock: 'Clear fallback lock files on ' + slot.toUpperCase() + '?\n\nOnly do this when no fallback process is running.',
};
// vvConfirm, never native confirm() — a native dialog offers "prevent additional dialogs",
// which kills every later dialog on the page document-wide. It returns a PROMISE, not a
// callback: passing a function would land in its opts argument and the body would never run,
// leaving a button that silently does nothing.
if (!await vvConfirm(labels[action] || ('Run ' + action + '?'))) return;
_vvFbBusy[slot] = true;
const out = document.getElementById('vv-fb-out-' + slot);
if (out) { out.style.color = '#555'; out.textContent = 'Running ' + action + '…'; }
document.querySelectorAll('.vv-fb-btn').forEach(b => { b.disabled = true; });
const fd = new URLSearchParams();
fd.append('action', action);
fd.append('host', slot);
fetch('/plugins/varaverk/api/fallback_control.php', { method: 'POST', body: fd })
.then(r => r.json())
.then(d => {
if (out) {
// The scripts explain refusals in words — "did not exit within 30s", "NOT
// force-killing" — so the reply is shown verbatim rather than reduced to ok/failed.
out.style.color = d.ok ? '#555' : '#ef5350';
out.textContent = d.ok ? (d.output || 'done') : (d.error || 'failed');
}
})
.catch(e => { if (out) { out.style.color = '#ef5350'; out.textContent = 'Request failed: ' + e; } })
.finally(() => {
_vvFbBusy[slot] = false;
document.querySelectorAll('.vv-fb-btn').forEach(b => { b.disabled = false; });
// Re-poll rather than guessing: whether the daemon actually stopped is a fact to read
// back, not one to infer from the request having returned.
setTimeout(vvFbLoad, 1200);
});
};
window.vvFbToggle = function(track, key) {
const on = !track.classList.contains('on');
track.classList.toggle('on', on);