728 lines
34 KiB
PHP
728 lines
34 KiB
PHP
<?php
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
// PURPOSE
|
|
// Fallback tab. Current state of every node (NORMAL / FALLBACK / NO_INTERNET / DARK),
|
|
// which tiers have activated, handback strike progress, and which covered containers are
|
|
// actually running.
|
|
//
|
|
// DESIGN PRINCIPLES
|
|
// Pure view — markup plus a poll against api/fallback.php. State parsing lives in
|
|
// include/fallback.php.
|
|
//
|
|
// Reports what each node believes about itself. Remote state is read from the partner's own
|
|
// state file rather than inferred from what this host can reach.
|
|
//
|
|
// Polls faster than most tabs (30s) because this is the page someone opens during an
|
|
// outage, when stale numbers are actively misleading.
|
|
//
|
|
// OPERATIONAL SAFEGUARDS
|
|
// Cannot cause a failover. The page stops daemons, starts a PREVIEW, and edits conf — it has
|
|
// no control that enters FALLBACK or forces a handback. Fallback is driven by fallback.sh
|
|
// reacting to real reachability, and a manual override from a browser is exactly the wrong
|
|
// way to enter that state. The one start control is --dry-run, which changes nothing.
|
|
//
|
|
// A missing state file renders as unknown, never as NORMAL — claiming healthy for a
|
|
// fallback process that is not running would be the worst possible error on this page.
|
|
//
|
|
// State age is shown next to state, always. Every value in the state file was written by a
|
|
// daemon that may not be running: this host displayed NORMAL from a file five days stale
|
|
// with no process alive, and nothing on the page said so.
|
|
//
|
|
// Stopping the test is a different button from stopping the daemon, deliberately. The test
|
|
// holds an iptables rule that only its own EXIT trap removes, so the two cannot share a
|
|
// control that might escalate to SIGKILL.
|
|
//
|
|
// RENDERS
|
|
// Per-node state and freshness, daemon liveness and mode, reachability legs, tier activation
|
|
// and delays, handback strikes, covered container status, quick settings, assistant
|
|
//
|
|
// DEPENDS ON
|
|
// api/fallback.php polled every 30s → include/fallback.php
|
|
// api/fallback_control.php start/stop actions → Fallback/fallback*.sh
|
|
// api/confform.php inline conf edits → include/confform.php
|
|
require_once dirname(__DIR__) . '/include/confui.php';
|
|
require_once dirname(__DIR__) . '/include/ai_chat.php';
|
|
|
|
// Same shared surface as the Partnership and Monitor tabs, gated the same way. Fallback
|
|
// questions — why a tier has not fired, what the mirror would actually start, whether a stale
|
|
// state file matters — are asked while looking at this page.
|
|
if (vv_ai_ui_on()) vv_ai_chat_assets();
|
|
?>
|
|
<style>
|
|
/* ── Host card: daemon, freshness, reachability, controls ── */
|
|
/* Class names are all vv-fb-* prefixed. Unraid Connect injects a global Tailwind layer into
|
|
every page, so a bare utility-shaped name like `fixed` or `grid` would be captured by it. */
|
|
.vv-fb-hcard { background:#141414;border:1px solid #262626;border-radius:6px;padding:12px 13px;min-width:0; }
|
|
.vv-fb-hcard.me { border-color:#2a3a2a; }
|
|
.vv-fb-hcard.warn { border-color:#4a3800; }
|
|
.vv-fb-hrow { display:flex;align-items:center;gap:7px;margin-bottom:8px;flex-wrap:wrap; }
|
|
.vv-fb-hid { font-size:13px;font-weight:700;color:#ddd; }
|
|
.vv-fb-hnm { font-size:11px;color:#666;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;min-width:0; }
|
|
.vv-fb-usbadge { background:#1a3a1a;color:#4caf50;font-size:8px;padding:1px 5px;border-radius:3px; }
|
|
|
|
.vv-fb-stats { display:grid;grid-template-columns:auto 1fr;gap:3px 10px;font-size:11px;margin-bottom:9px; }
|
|
.vv-fb-stats b { color:#444;font-weight:400; }
|
|
.vv-fb-sv { color:#999;min-width:0; }
|
|
.vv-fb-sv.good { color:#4caf50; }
|
|
.vv-fb-sv.warn { color:#ffb74d; }
|
|
.vv-fb-sv.bad { color:#ef5350; }
|
|
.vv-fb-sv.dim { color:#3a3a3a; }
|
|
|
|
/* Reachability legs — separate facts, never rolled into one boolean. */
|
|
.vv-fb-legs { display:flex;gap:5px;flex-wrap:wrap;margin-bottom:9px; }
|
|
.vv-fb-leg { font-size:9px;padding:1px 6px;border-radius:3px;border:1px solid #222;background:#111;color:#444; }
|
|
.vv-fb-leg.ok { border-color:#243a24;background:#101a10;color:#4caf50; }
|
|
.vv-fb-leg.no { border-color:#3a1e1e;background:#1a1010;color:#ef5350; }
|
|
|
|
.vv-fb-acts { display:flex;gap:6px;flex-wrap:wrap;margin-top:9px;padding-top:8px;border-top:1px solid #1e1e1e; }
|
|
.vv-fb-btn { background:#111;border:1px solid #262626;color:#888;border-radius:3px;
|
|
padding:4px 9px;cursor:pointer;font-size:10px;white-space:nowrap; }
|
|
.vv-fb-btn:hover:not(:disabled) { border-color:#3a3a3a;color:#ccc; }
|
|
.vv-fb-btn:disabled { opacity:.4;cursor:default; }
|
|
.vv-fb-btn.stop { color:#ef9a9a; }
|
|
.vv-fb-btn.go { color:#4a9eff; }
|
|
.vv-fb-out { font-size:9px;color:#555;margin-top:6px;white-space:pre-wrap;word-break:break-word;
|
|
max-height:80px;overflow:auto;font-family:monospace; }
|
|
|
|
/* Quick settings — one card per setting rather than one card of rows, so each carries its own
|
|
why. The old single card listed three numbers with no indication what changing one costs. */
|
|
.vv-fb-qs { display:grid;grid-template-columns:repeat(auto-fit,minmax(190px,1fr));gap:10px;margin-bottom:12px; }
|
|
.vv-fb-qcard { background:#141414;border:1px solid #242424;border-radius:6px;padding:10px 11px; }
|
|
.vv-fb-qlbl { font-size:11px;color:#aaa;font-weight:600;margin-bottom:2px; }
|
|
.vv-fb-qsub { font-size:9px;color:#3f3f3f;line-height:1.35;margin-bottom:7px; }
|
|
.vv-fb-qin { display:flex;align-items:center;gap:7px; }
|
|
/* ── Existing status styles ── */
|
|
.vv-fb-active { background:#1a1200;border:1px solid #5a3800;border-radius:6px;padding:12px 14px; }
|
|
.vv-fb-active-h { display:flex;align-items:baseline;gap:10px;margin-bottom:8px; }
|
|
.vv-fb-badge { font-size:11px;font-weight:bold;letter-spacing:.06em;padding:2px 7px;border-radius:3px;flex-shrink:0; }
|
|
.vv-fb-badge.fb { background:#5a3800;color:#ffb74d; }
|
|
.vv-fb-badge.norm { background:#1a2a1a;color:#4caf50; }
|
|
.vv-fb-badge.dark { background:#2a1a2a;color:#9c27b0; }
|
|
.vv-fb-badge.nonet{ background:#1a1a2a;color:#5c7cfa; }
|
|
.vv-fb-badge.susp { background:#2a1a00;color:#888; }
|
|
.vv-fb-meta { display:flex;gap:18px;flex-wrap:wrap;margin-bottom:10px; }
|
|
.vv-fb-meta-item{ display:flex;flex-direction:column;gap:1px; }
|
|
.vv-fb-meta-val { font-size:17px;font-weight:bold;color:#ffb74d; }
|
|
.vv-fb-meta-lbl { font-size:10px;color:#5a4020; }
|
|
.vv-fb-ctrs { display:flex;gap:6px;flex-wrap:wrap;margin-top:6px; }
|
|
.vv-fb-ctr { font-size:11px;padding:2px 8px;border-radius:3px;background:#2a1e00;color:#ffb74d;border:1px solid #4a2e00; }
|
|
.vv-fb-ctr.running { background:#1a2a1a;color:#6fcf97;border-color:#2d4a2d; }
|
|
.vv-fb-ctr.stopped { background:#2a1a1a;color:#e57;border-color:#4a2020;opacity:.7; }
|
|
.vv-fb-node { grid-column:span 4; }
|
|
.vv-fb-node-h { display:flex;align-items:baseline;gap:8px;margin-bottom:10px; }
|
|
.vv-fb-node-id { font-size:12px;font-weight:bold;color:#666;letter-spacing:.06em;text-transform:uppercase; }
|
|
.vv-fb-node-nm { font-size:11px;color:#3a3a3a; }
|
|
.vv-fb-arrow { font-size:11px;color:#333; }
|
|
.vv-fb-covers { font-size:11px;color:#3a3a3a; }
|
|
.vv-fb-tier { margin-bottom:8px; }
|
|
.vv-fb-tier-h { display:flex;align-items:baseline;gap:6px;margin-bottom:4px; }
|
|
.vv-fb-tier-lbl { font-size:11px;font-weight:bold;color:#555;text-transform:uppercase;letter-spacing:.04em; }
|
|
.vv-fb-tier-delay { font-size:10px;color:#3a3a3a; }
|
|
.vv-fb-tier-pills { display:flex;gap:5px;flex-wrap:wrap; }
|
|
.vv-fb-pill { font-size:11px;padding:2px 7px;border-radius:3px;background:#1e1e1e;color:#777;border:1px solid #2a2a2a; }
|
|
.vv-fb-pill.active-t { background:#1a2010;color:#8bc34a;border-color:#2d3a1d; }
|
|
.vv-fb-pill.empty { color:#333;font-style:italic; }
|
|
.vv-fb-state-dot { width:6px;height:6px;border-radius:50%;flex-shrink:0;margin-top:3px; }
|
|
.vv-fb-sep { border:none;border-top:1px solid #222;margin:8px 0; }
|
|
.vv-fb-disabled { grid-column:1/-1;color:#3a3a3a;font-size:12px;padding:20px 0;text-align:center; }
|
|
|
|
/* ── Controls + settings card ── */
|
|
.vv-fb-card { background:#161616;border:1px solid #2a2a2a;border-radius:6px;padding:14px 16px;margin-bottom:14px; }
|
|
.vv-fb-card-hdr { font-size:11px;font-weight:700;color:#666;text-transform:uppercase;letter-spacing:.07em;margin-bottom:12px; }
|
|
.vv-fb-ctrl-row { display:flex;justify-content:space-between;align-items:center;gap:12px;padding:5px 0; }
|
|
.vv-fb-ctrl-lbl { font-size:12px;color:#888; }
|
|
.vv-fb-ctrl-sub { font-size:10px;color:#3a3a3a;margin-top:2px; }
|
|
.vv-fb-tog { width:32px;height:18px;border-radius:9px;background:#222;border:1px solid #333;
|
|
position:relative;transition:background .15s,border-color .15s;flex-shrink:0;cursor:pointer; }
|
|
.vv-fb-tog.on { background:#1a3a1a;border-color:#2d5a2d; }
|
|
.vv-fb-tog::after { content:'';position:absolute;top:2px;left:2px;width:12px;height:12px;
|
|
border-radius:50%;background:#555;transition:left .15s,background .15s; }
|
|
.vv-fb-tog.on::after { left:16px;background:#4caf50; }
|
|
.vv-fb-set-row { display:flex;justify-content:space-between;align-items:center;padding:5px 0; }
|
|
.vv-fb-set-lbl { font-size:11px;color:#555; }
|
|
.vv-fb-set-inp { background:#0d0d0d;border:1px solid #2a2a2a;border-radius:3px;color:#bbb;
|
|
font-size:12px;padding:4px 8px;outline:none;width:72px;font-family:monospace;
|
|
text-align:right;box-sizing:border-box; }
|
|
.vv-fb-set-inp:focus { border-color:#444; }
|
|
.vv-fb-set-unit { font-size:10px;color:#3a3a3a;min-width:56px; }
|
|
.vv-fb-save-btn { background:#1a1a1a;border:1px solid #333;color:#888;font-size:11px;
|
|
padding:5px 14px;border-radius:3px;cursor:pointer; }
|
|
.vv-fb-save-btn:hover { border-color:#555;color:#ccc; }
|
|
.vv-fb-save-btn:disabled { opacity:.4;cursor:default; }
|
|
</style>
|
|
|
|
<!-- Controls card -->
|
|
<div class="vv-fb-card">
|
|
<div class="vv-fb-card-hdr">Controls</div>
|
|
|
|
<div class="vv-fb-ctrl-row">
|
|
<div>
|
|
<div class="vv-fb-ctrl-lbl">Partnership</div>
|
|
<div class="vv-fb-ctrl-sub">Master gate — disabling stops all cross-server operations</div>
|
|
</div>
|
|
<div class="vv-fb-tog" id="vv-fb-pt-tog" onclick="vvFbToggle(this,'PARTNERSHIP_ENABLED')"></div>
|
|
</div>
|
|
|
|
<hr class="vv-fb-sep">
|
|
|
|
<div class="vv-fb-ctrl-row">
|
|
<div>
|
|
<div class="vv-fb-ctrl-lbl">Fallback</div>
|
|
<div class="vv-fb-ctrl-sub">Mutual container failover between nodes</div>
|
|
</div>
|
|
<div class="vv-fb-tog" id="vv-fb-en-tog" onclick="vvFbToggle(this,'FALLBACK_ENABLED')"></div>
|
|
</div>
|
|
|
|
<hr class="vv-fb-sep">
|
|
|
|
<div class="vv-fb-ctrl-row">
|
|
<div>
|
|
<div class="vv-fb-ctrl-lbl">Rsync on handback</div>
|
|
<div class="vv-fb-ctrl-sub">Writeback rsync when the covered host recovers and containers return</div>
|
|
</div>
|
|
<div class="vv-fb-tog" id="vv-fb-rsync-tog" onclick="vvFbToggle(this,'FALLBACK_RSYNC_ENABLED')"></div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Mesh verdict — one line answering "is this armed and is anything watching" before any detail -->
|
|
<div class="vv-fb-card" id="vv-fb-verdict-card" style="margin-bottom:12px;">
|
|
<div id="vv-fb-verdict" style="font-size:12px;color:#555;">Loading…</div>
|
|
</div>
|
|
|
|
<!-- Status section -->
|
|
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;padding:0 2px;">
|
|
<span style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;">Nodes</span>
|
|
<span style="font-size:11px;color:#3a3a3a;" id="vv-fb-ts"></span>
|
|
</div>
|
|
|
|
<div id="vv-fb-grid" style="display:grid;grid-template-columns:repeat(auto-fit,minmax(340px,1fr));gap:12px;margin-bottom:14px;">
|
|
<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">Loading…</div>
|
|
</div>
|
|
|
|
<!-- Assistant + mesh chat -->
|
|
<?php if (vv_ai_ui_on()): ?>
|
|
<div class="vv-card" id="vv-fb-ai-card" style="margin-bottom:12px;">
|
|
<?php vv_ai_chat_markup('vv-fb-ai', [
|
|
'profile' => 'varaverk',
|
|
'compact' => true,
|
|
// Fallback is the mesh's business by definition — a failover is two hosts agreeing about
|
|
// each other — so the mesh side leads here, as it does on Partnership.
|
|
'mesh' => true,
|
|
'meshDefault' => true,
|
|
'title' => 'Assistant',
|
|
'height' => '300px',
|
|
'tall' => '500px',
|
|
'tallLarge' => '750px',
|
|
'empty' => 'Ask about fallback — why a tier has not fired, what the partner would '
|
|
. 'start if this host went dark, whether a stale state file matters.',
|
|
]); ?>
|
|
</div>
|
|
<?php endif; ?>
|
|
|
|
<!-- Quick settings -->
|
|
<div style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;margin-bottom:10px;padding:0 2px;">Quick settings</div>
|
|
|
|
<div class="vv-fb-qs">
|
|
<div class="vv-fb-qcard">
|
|
<div class="vv-fb-qlbl">Check interval</div>
|
|
<div class="vv-fb-qsub">How often the daemon re-tests the partner. Also the age at which a state file is stale.</div>
|
|
<div class="vv-fb-qin">
|
|
<input class="vv-fb-set-inp" id="vv-fb-interval" type="number" min="5" max="300">
|
|
<span class="vv-fb-set-unit">seconds</span>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="vv-fb-qcard">
|
|
<div class="vv-fb-qlbl">Handback strikes</div>
|
|
<div class="vv-fb-qsub">Consecutive healthy checks before handing services back. Higher rides out a flapping link.</div>
|
|
<div class="vv-fb-qin">
|
|
<input class="vv-fb-set-inp" id="vv-fb-strikes" type="number" min="1" max="20">
|
|
<span class="vv-fb-set-unit">consecutive</span>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="vv-fb-qcard">
|
|
<div class="vv-fb-qlbl">Partnership suspend after</div>
|
|
<div class="vv-fb-qsub">Minutes without an active partnership before fallback suspends itself. 0 suspends immediately.</div>
|
|
<div class="vv-fb-qin">
|
|
<input class="vv-fb-set-inp" id="vv-fb-suspend" type="number" min="0" max="1440">
|
|
<span class="vv-fb-set-unit">minutes</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div style="display:flex;justify-content:flex-end;align-items:center;gap:10px;margin-bottom:14px;">
|
|
<span id="vv-fb-set-fb" style="font-size:11px;"></span>
|
|
<button class="vv-fb-save-btn" id="vv-fb-save-btn" onclick="vvFbSaveSettings()">Save</button>
|
|
</div>
|
|
|
|
<?php
|
|
// The conf sections this page is about, drawn by the shared renderer. They were reachable
|
|
// only from the Settings tab's catch-all, which is a long way to go for a setting named
|
|
// after the page you are already looking at.
|
|
vv_conf_ui_card('vv-cf-fallback', 'fallback|tier', 'Fallback settings');
|
|
?>
|
|
<script>
|
|
(function() {
|
|
|
|
function _dur(startTs) {
|
|
if (!startTs || startTs === 0) return '—';
|
|
const s = Math.floor(Date.now() / 1000) - startTs;
|
|
if (s < 60) return s + 's';
|
|
if (s < 3600) return Math.floor(s / 60) + 'm';
|
|
const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60);
|
|
return m ? h + 'h ' + m + 'm' : h + 'h';
|
|
}
|
|
|
|
function _fmtDelay(min) {
|
|
if (!min) return 'immediate';
|
|
if (min < 60) return min + 'min';
|
|
const h = Math.floor(min / 60), m = min % 60;
|
|
return m ? h + 'h ' + m + 'm' : h + 'h';
|
|
}
|
|
|
|
function _activeTier(state) {
|
|
if (!state) return 0;
|
|
if (state.tier4_started) return 4;
|
|
if (state.tier3_started) return 3;
|
|
if (state.tier2_started) return 2;
|
|
return 1;
|
|
}
|
|
|
|
function _stateBadge(st) {
|
|
const map = {
|
|
NORMAL: ['norm', 'NORMAL'],
|
|
FALLBACK: ['fb', 'FALLBACK'],
|
|
NO_INTERNET: ['nonet', 'NO INTERNET'],
|
|
DARK: ['dark', 'DARK'],
|
|
OFFLINE: ['dark', 'OFFLINE'],
|
|
UNREACHABLE: ['dark', 'UNREACHABLE'],
|
|
UNKNOWN: ['dark', 'UNKNOWN'],
|
|
SUSPENDED: ['susp', 'SUSPENDED'],
|
|
};
|
|
const [cls, label] = map[st] || ['dark', st];
|
|
return `<span class="vv-fb-badge ${cls}">${label}</span>`;
|
|
}
|
|
|
|
function _stateDot(st) {
|
|
const col = {
|
|
NORMAL:'#4caf50', FALLBACK:'#ffb74d',
|
|
NO_INTERNET:'#5c7cfa', DARK:'#9c27b0',
|
|
OFFLINE:'#555', UNREACHABLE:'#555', UNKNOWN:'#333', SUSPENDED:'#444',
|
|
}[st] || '#333';
|
|
return `<span class="vv-fb-state-dot" style="background:${col}"></span>`;
|
|
}
|
|
|
|
function _activeCard(nodes, handbackReq) {
|
|
const active = nodes.filter(n => n.state && n.state.state === 'FALLBACK');
|
|
if (!active.length) return '';
|
|
|
|
return active.map(covering => {
|
|
const st = covering.state;
|
|
const tier = _activeTier(st);
|
|
const cov = covering.covers;
|
|
const covered = cov ? cov.hostname : '?';
|
|
|
|
let expected = [...(cov?.tier1 || [])];
|
|
if (tier >= 2) expected = expected.concat(cov?.tier2 || []);
|
|
if (tier >= 3) expected = expected.concat(cov?.tier3 || []);
|
|
if (tier >= 4) expected = expected.concat(cov?.tier4 || []);
|
|
const runningSet = new Set(covering.running || []);
|
|
|
|
const ctrPills = expected.length
|
|
? expected.map(c => {
|
|
const cls = runningSet.has(c) ? 'running' : 'stopped';
|
|
const sym = runningSet.has(c) ? '▲' : '▼';
|
|
return `<span class="vv-fb-ctr ${cls}">${sym} ${c}</span>`;
|
|
}).join('')
|
|
: '<span style="color:#5a4020;font-size:11px;">No containers configured for this tier</span>';
|
|
|
|
return `<div class="vv-card vv-fb-active" style="grid-column:1/-1;">
|
|
<div class="vv-fb-active-h">
|
|
${_stateBadge('FALLBACK')}
|
|
<span style="font-size:12px;color:#aa7020;">${covering.id} covering ${cov?.id || '?'} (${covered})</span>
|
|
</div>
|
|
<div class="vv-fb-meta">
|
|
<div class="vv-fb-meta-item">
|
|
<span class="vv-fb-meta-val">${_dur(st.fallback_start)}</span>
|
|
<span class="vv-fb-meta-lbl">DURATION</span>
|
|
</div>
|
|
<div class="vv-fb-meta-item">
|
|
<span class="vv-fb-meta-val">Tier ${tier}</span>
|
|
<span class="vv-fb-meta-lbl">ACTIVE TIER</span>
|
|
</div>
|
|
<div class="vv-fb-meta-item">
|
|
<span class="vv-fb-meta-val">${st.handback_strikes} / ${handbackReq}</span>
|
|
<span class="vv-fb-meta-lbl">HANDBACK STRIKES</span>
|
|
</div>
|
|
<div class="vv-fb-meta-item">
|
|
<span class="vv-fb-meta-val">${expected.length}</span>
|
|
<span class="vv-fb-meta-lbl">CONTAINERS</span>
|
|
</div>
|
|
</div>
|
|
<div class="vv-fb-ctrs">${ctrPills}</div>
|
|
</div>`;
|
|
}).join('');
|
|
}
|
|
|
|
function _tierSection(tiers, activeTier, delays) {
|
|
const defs = [
|
|
{ n: 1, key: 'tier1', label: 'Tier 1', delay: 0 },
|
|
{ n: 2, key: 'tier2', label: 'Tier 2', delay: delays?.tier2 },
|
|
{ n: 3, key: 'tier3', label: 'Tier 3', delay: delays?.tier3 },
|
|
{ n: 4, key: 'tier4', label: 'Tier 4', delay: delays?.tier4 },
|
|
];
|
|
return defs.map(({ n, key, label, delay }) => {
|
|
const containers = tiers[key] || [];
|
|
const isActive = activeTier >= n;
|
|
const pills = containers.length
|
|
? containers.map(c => `<span class="vv-fb-pill${isActive ? ' active-t' : ''}">${c}</span>`).join('')
|
|
: `<span class="vv-fb-pill empty">none</span>`;
|
|
const delayStr = n === 1 ? 'immediate' : _fmtDelay(delay);
|
|
return `<div class="vv-fb-tier">
|
|
<div class="vv-fb-tier-h">
|
|
<span class="vv-fb-tier-lbl">${label}</span>
|
|
<span class="vv-fb-tier-delay">${delayStr}</span>
|
|
</div>
|
|
<div class="vv-fb-tier-pills">${pills}</div>
|
|
</div>`;
|
|
}).join('');
|
|
}
|
|
|
|
// Only rendered against FRESH state — see _nodeCard. A grace timer counted off a state file
|
|
// nobody has written for five days reads as an active grace window, which is precisely the kind
|
|
// of stale-as-current claim this page must not make.
|
|
function _ptStatus(st) {
|
|
if (!st) return '';
|
|
if (st.partnership_suspended) return _stateBadge('SUSPENDED');
|
|
if (st.partner_lost_at && st.partner_lost_at > 0) {
|
|
const minGone = Math.floor((Date.now() / 1000 - st.partner_lost_at) / 60);
|
|
return `<span class="vv-fb-badge susp">GRACE ${minGone}m</span>`;
|
|
}
|
|
return '';
|
|
}
|
|
|
|
function _age(sec) {
|
|
if (sec === null || sec === undefined) return '—';
|
|
if (sec < 60) return sec + 's ago';
|
|
if (sec < 3600) return Math.floor(sec / 60) + 'm ago';
|
|
if (sec < 86400) return Math.floor(sec / 3600) + 'h ago';
|
|
return Math.floor(sec / 86400) + 'd ago';
|
|
}
|
|
|
|
// Daemon liveness, stated as one of four distinct things rather than a boolean. "Not running"
|
|
// and "running a preview" are opposite operational facts and the old page showed neither.
|
|
function _daemonRow(p, checkInterval) {
|
|
if (!p) return ['dim', 'unknown', ''];
|
|
if (p.stale_lock) return ['bad', 'not running · stale lock' + (p.pid ? ' (PID ' + p.pid + ')' : ''), 'stale'];
|
|
if (p.running === null) return ['dim', 'unknown — partner not reachable', ''];
|
|
if (!p.running) return ['bad', 'not running', ''];
|
|
const mode = p.mode === 'dry-run'
|
|
? '<span style="color:#4a9eff;">dry run</span>'
|
|
: '<span style="color:#4caf50;">live</span>';
|
|
return ['good', mode + ' · PID ' + p.pid, 'up'];
|
|
}
|
|
|
|
// State freshness. The daemon rewrites this file every check interval, so anything past a few
|
|
// intervals is a wedged process or a dead one — and the state value it left behind is a claim
|
|
// about the past, not the present.
|
|
function _freshness(node, checkInterval) {
|
|
const a = node.state_age;
|
|
// A dry run redirects its state writes to a per-PID copy so the live file is never touched.
|
|
// That is the point of it — but it means the live file goes on ageing while a preview runs,
|
|
// and "daemon: dry run" beside "state age: 5d — stale" reads as a fault when it is the
|
|
// designed behaviour. Say which it is rather than leaving the operator to chase it.
|
|
const dry = node.proc && node.proc.running && node.proc.mode === 'dry-run';
|
|
|
|
if (a === null || a === undefined) {
|
|
if (dry) return ['dim', 'none — dry run does not write live state'];
|
|
return node.is_me
|
|
? ['bad', 'no state file — fallback has never run here']
|
|
: ['dim', 'not readable from here'];
|
|
}
|
|
const limit = Math.max(120, (checkInterval || 30) * 4);
|
|
if (a > limit && dry) return ['dim', _age(a) + ' — not refreshed: dry run writes elsewhere'];
|
|
return [a > limit ? 'bad' : 'good', _age(a) + (a > limit ? ' — stale' : '')];
|
|
}
|
|
|
|
function _leg(ok, label) {
|
|
const cls = ok === true ? 'ok' : ok === false ? 'no' : '';
|
|
const sym = ok === true ? '✓' : ok === false ? '✕' : '·';
|
|
return `<span class="vv-fb-leg ${cls}">${sym} ${label}</span>`;
|
|
}
|
|
|
|
function _nodeCard(node, data) {
|
|
const st = node.state || {};
|
|
const state = st.partnership_suspended ? 'SUSPENDED' : (st.state || 'UNKNOWN');
|
|
const cov = node.covers;
|
|
const active = _activeTier(state === 'FALLBACK' ? st : null);
|
|
const slot = node.slot;
|
|
|
|
const [dCls, dTxt, dKind] = _daemonRow(node.proc, data.check_interval);
|
|
const [fCls, fTxt] = _freshness(node, data.check_interval);
|
|
const test = node.proc_test || {};
|
|
const reach = node.reach || {};
|
|
|
|
const covTarget = cov
|
|
? `<span class="vv-fb-sv">${cov.id} <span style="color:#3a3a3a;">(${vvEscHtml(cov.hostname)})</span></span>`
|
|
: '<span class="vv-fb-sv dim">nothing configured</span>';
|
|
|
|
const tierSection = cov
|
|
? _tierSection(cov, active, cov.delays)
|
|
: '<div style="color:#3a3a3a;font-size:11px;">No coverage configured</div>';
|
|
|
|
// Tier lists live in the COVERED host's conf, so an empty set here is a config gap on the
|
|
// other side, not on this one. Saying which conf to edit saves the hunt.
|
|
const tierTotal = cov ? ['tier1','tier2','tier3','tier4'].reduce((n,k)=>n+((cov[k]||[]).length),0) : 0;
|
|
const tierWarn = (cov && tierTotal === 0)
|
|
? `<div style="font-size:9px;color:#5a4020;margin-top:5px;">No containers in any tier — set FALLBACK_${cov.id}_TIER1 in ${cov.slot}.conf</div>`
|
|
: '';
|
|
|
|
const busy = _vvFbBusy[slot] ? 'disabled' : '';
|
|
const acts = `
|
|
<button class="vv-fb-btn stop" ${busy} onclick="vvFbAct('${slot}','stop',this)"
|
|
title="fallback.sh --stop — SIGTERM, then SIGKILL after 10s">Stop fallback</button>
|
|
<button class="vv-fb-btn go" ${busy} onclick="vvFbAct('${slot}','start_dry',this)"
|
|
title="fallback.sh --dry-run — previews decisions, changes nothing">Start dry run</button>
|
|
<button class="vv-fb-btn stop" ${busy} onclick="vvFbAct('${slot}','stop_test',this)"
|
|
title="fallback_test.sh --stop — SIGTERM only, never SIGKILL: only its trap removes the iptables rule">Stop test</button>
|
|
${(node.proc?.stale_lock || test.stale_lock)
|
|
? `<button class="vv-fb-btn" ${busy} onclick="vvFbAct('${slot}','clear_lock',this)"
|
|
title="Remove lock files left by a killed run">Clear lock</button>` : ''}`;
|
|
|
|
const testRow = test.running
|
|
? `<b>Test</b><span class="vv-fb-sv warn">running · PID ${test.pid} — holds an iptables rule</span>`
|
|
: test.stale_lock
|
|
? `<b>Test</b><span class="vv-fb-sv bad">stale lock — an iptables rule may be stranded</span>`
|
|
: '';
|
|
|
|
return `<div class="vv-fb-hcard${node.is_me ? ' me' : ''}${(dCls === 'bad' || fCls === 'bad') ? ' warn' : ''}">
|
|
<div class="vv-fb-hrow">
|
|
${_stateDot(state)}
|
|
<span class="vv-fb-hid">${node.id}</span>
|
|
<span class="vv-fb-hnm">${vvEscHtml(node.hostname)}</span>
|
|
${node.is_me ? '<span class="vv-fb-usbadge">US</span>' : ''}
|
|
<span style="flex:1"></span>
|
|
${(node.is_me && fCls === 'good') ? _ptStatus(st) : ''}
|
|
${_stateBadge(state)}
|
|
</div>
|
|
|
|
<div class="vv-fb-legs">
|
|
${_leg(reach.tailscale, 'tailscale')}
|
|
${_leg(node.is_me ? true : reach.ssh, 'ssh')}
|
|
${_leg(reach.state_file, 'state file')}
|
|
${node.ts_ip ? `<span class="vv-fb-leg">${vvEscHtml(node.ts_ip)}</span>` : ''}
|
|
</div>
|
|
|
|
<div class="vv-fb-stats">
|
|
<b>Daemon</b> <span class="vv-fb-sv ${dCls}">${dTxt}</span>
|
|
<b>State age</b> <span class="vv-fb-sv ${fCls}">${fTxt}</span>
|
|
<b>Covers</b> ${covTarget}
|
|
<b>Running</b> <span class="vv-fb-sv">${node.running_count ?? 0} containers</span>
|
|
${state === 'FALLBACK'
|
|
? `<b>Strikes</b><span class="vv-fb-sv warn">${st.handback_strikes} / ${data.handback_req || 3}</span>` : ''}
|
|
${testRow}
|
|
</div>
|
|
|
|
<hr class="vv-fb-sep">
|
|
${tierSection}
|
|
${tierWarn}
|
|
|
|
<div class="vv-fb-acts">${acts}</div>
|
|
<div class="vv-fb-out" id="vv-fb-out-${slot}"></div>
|
|
</div>`;
|
|
}
|
|
|
|
// One mesh-level sentence, above the detail. Answers the two questions the page exists for
|
|
// before anyone has to read a card: is fallback armed, and is anything actually watching.
|
|
function _verdict(data) {
|
|
const el = document.getElementById('vv-fb-verdict');
|
|
if (!el) return;
|
|
const nodes = data.nodes || [];
|
|
if (!data.partnership_enabled)
|
|
return void (el.innerHTML = '<span style="color:#ef5350;">PARTNERSHIP_ENABLED=false</span> — all cross-server operations are disabled.');
|
|
if (!data.fb_enabled)
|
|
return void (el.innerHTML = '<span style="color:#ffb74d;">FALLBACK_ENABLED=false</span> — nothing will fail over. Arming it takes effect at the next array start.');
|
|
|
|
const inFb = nodes.filter(n => (n.state?.state) === 'FALLBACK');
|
|
const live = nodes.filter(n => n.proc?.running && n.proc.mode === 'live');
|
|
const dry = nodes.filter(n => n.proc?.running && n.proc.mode === 'dry-run');
|
|
const dead = nodes.filter(n => n.proc && n.proc.running === false);
|
|
|
|
if (inFb.length)
|
|
return void (el.innerHTML = `<span style="color:#ffb74d;font-weight:600;">${inFb.map(n=>n.id).join(', ')} in FALLBACK</span> — covering for a partner right now.`);
|
|
|
|
let s = `<span style="color:#4caf50;">Armed</span> · ${live.length}/${nodes.length} node${nodes.length!==1?'s':''} running the daemon`;
|
|
if (dry.length) s += ` · <span style="color:#4a9eff;">${dry.length} in dry run</span>`;
|
|
if (dead.length) s += ` · <span style="color:#ef5350;">${dead.map(n=>n.id).join(', ')} not running — nothing would detect an outage there</span>`;
|
|
el.innerHTML = s;
|
|
}
|
|
function _setToggles(data) {
|
|
const pairs = [
|
|
['vv-fb-pt-tog', !!data.partnership_enabled],
|
|
['vv-fb-en-tog', !!data.fb_enabled],
|
|
['vv-fb-rsync-tog', !!data.fb_rsync_enabled],
|
|
];
|
|
pairs.forEach(([id, on]) => {
|
|
const el = document.getElementById(id);
|
|
if (el) el.classList.toggle('on', on);
|
|
});
|
|
}
|
|
|
|
function _setInputs(data) {
|
|
const fields = [
|
|
['vv-fb-interval', data.check_interval ?? 30],
|
|
['vv-fb-strikes', data.handback_req ?? 3],
|
|
['vv-fb-suspend', data.suspend_after ?? 120],
|
|
];
|
|
fields.forEach(([id, val]) => {
|
|
const el = document.getElementById(id);
|
|
if (el && el !== document.activeElement) el.value = val;
|
|
});
|
|
}
|
|
|
|
function _render(data) {
|
|
_vvFbLast = data;
|
|
_setToggles(data);
|
|
_setInputs(data);
|
|
_verdict(data);
|
|
|
|
const grid = document.getElementById('vv-fb-grid');
|
|
|
|
if (!data.partnership_enabled) {
|
|
grid.innerHTML = '<div class="vv-fb-disabled">PARTNERSHIP_ENABLED=false — all cross-server operations disabled</div>';
|
|
} else if (!data.fb_enabled) {
|
|
grid.innerHTML = '<div class="vv-fb-disabled">FALLBACK_ENABLED=false — fallback monitoring is disabled</div>';
|
|
} else {
|
|
const nodes = data.nodes || [];
|
|
let html = _activeCard(nodes, data.handback_req || 3);
|
|
for (const node of nodes) html += _nodeCard(node, data);
|
|
grid.innerHTML = html || '<div class="vv-fb-disabled">No nodes configured.</div>';
|
|
}
|
|
|
|
const ts = data.ts
|
|
? new Date(data.ts * 1000).toLocaleString([], {
|
|
month:'numeric', day:'numeric', year:'numeric',
|
|
hour:'2-digit', minute:'2-digit', second:'2-digit'})
|
|
: '';
|
|
document.getElementById('vv-fb-ts').textContent = ts ? 'Updated: ' + ts : '';
|
|
}
|
|
|
|
function vvFbLoad() {
|
|
fetch('/plugins/varaverk/api/fallback.php')
|
|
.then(r => r.json())
|
|
.then(_render)
|
|
.catch(() => {
|
|
document.getElementById('vv-fb-grid').innerHTML =
|
|
'<div style="grid-column:1/-1;color:#555;font-size:12px;padding:24px 0;text-align:center;">Error loading fallback data — check API</div>';
|
|
});
|
|
}
|
|
|
|
// Per-slot in-flight guard, so a second click cannot fire while a stop is still escalating.
|
|
// Keyed by slot rather than one page-wide flag: acting on one host must not freeze the other's
|
|
// controls, and on this page the two hosts are routinely in different states.
|
|
const _vvFbBusy = {};
|
|
let _vvFbLast = null;
|
|
|
|
// URLSearchParams, not FormData. Unraid's CSRF token injector is jQuery-only, and a multipart
|
|
// body from native fetch() hangs here with no status and no server-side trace.
|
|
window.vvFbAct = async function (slot, action, btn) {
|
|
if (_vvFbBusy[slot]) return;
|
|
|
|
const labels = {
|
|
stop: 'Stop the fallback daemon on ' + slot.toUpperCase() + '?',
|
|
stop_test: 'Stop the fallback TEST on ' + slot.toUpperCase() + '?\n\nSIGTERM only — the test needs its own trap to remove the iptables rule it installed.',
|
|
start_dry: 'Start a fallback DRY RUN on ' + slot.toUpperCase() + '?\n\nIt previews decisions and changes nothing.',
|
|
clear_lock: 'Clear fallback lock files on ' + slot.toUpperCase() + '?\n\nOnly do this when no fallback process is running.',
|
|
};
|
|
|
|
// vvConfirm, never native confirm() — a native dialog offers "prevent additional dialogs",
|
|
// which kills every later dialog on the page document-wide. It returns a PROMISE, not a
|
|
// callback: passing a function would land in its opts argument and the body would never run,
|
|
// leaving a button that silently does nothing.
|
|
if (!await vvConfirm(labels[action] || ('Run ' + action + '?'))) return;
|
|
|
|
_vvFbBusy[slot] = true;
|
|
const out = document.getElementById('vv-fb-out-' + slot);
|
|
if (out) { out.style.color = '#555'; out.textContent = 'Running ' + action + '…'; }
|
|
document.querySelectorAll('.vv-fb-btn').forEach(b => { b.disabled = true; });
|
|
|
|
const fd = new URLSearchParams();
|
|
fd.append('action', action);
|
|
fd.append('host', slot);
|
|
|
|
fetch('/plugins/varaverk/api/fallback_control.php', { method: 'POST', body: fd })
|
|
.then(r => r.json())
|
|
.then(d => {
|
|
if (out) {
|
|
// The scripts explain refusals in words — "did not exit within 30s", "NOT
|
|
// force-killing" — so the reply is shown verbatim rather than reduced to ok/failed.
|
|
out.style.color = d.ok ? '#555' : '#ef5350';
|
|
out.textContent = d.ok ? (d.output || 'done') : (d.error || 'failed');
|
|
}
|
|
})
|
|
.catch(e => { if (out) { out.style.color = '#ef5350'; out.textContent = 'Request failed: ' + e; } })
|
|
.finally(() => {
|
|
_vvFbBusy[slot] = false;
|
|
document.querySelectorAll('.vv-fb-btn').forEach(b => { b.disabled = false; });
|
|
// Re-poll rather than guessing: whether the daemon actually stopped is a fact to read
|
|
// back, not one to infer from the request having returned.
|
|
setTimeout(vvFbLoad, 1200);
|
|
});
|
|
};
|
|
|
|
window.vvFbToggle = function(track, key) {
|
|
const on = !track.classList.contains('on');
|
|
track.classList.toggle('on', on);
|
|
const fd = new URLSearchParams();
|
|
fd.append('id', 'fallback');
|
|
fd.append('changes', JSON.stringify([{ file: 'master.conf', key, value: on ? 'true' : 'false', type: 'scalar' }]));
|
|
fetch('/plugins/varaverk/api/confform.php', { method: 'POST', body: fd })
|
|
.then(r => r.json())
|
|
.then(d => { if (!d.ok) track.classList.toggle('on', !on); })
|
|
.catch(() => track.classList.toggle('on', !on));
|
|
};
|
|
|
|
window.vvFbSaveSettings = function() {
|
|
const ivEl = document.getElementById('vv-fb-interval');
|
|
const stEl = document.getElementById('vv-fb-strikes');
|
|
const suEl = document.getElementById('vv-fb-suspend');
|
|
const fb = document.getElementById('vv-fb-set-fb');
|
|
const btn = document.getElementById('vv-fb-save-btn');
|
|
|
|
const interval = parseInt(ivEl.value, 10);
|
|
const strikes = parseInt(stEl.value, 10);
|
|
const suspend = parseInt(suEl.value, 10);
|
|
|
|
if ([interval, strikes, suspend].some(n => isNaN(n) || n < 0)) {
|
|
fb.style.color = '#ef5350'; fb.textContent = 'Invalid values'; return;
|
|
}
|
|
|
|
btn.disabled = true; btn.textContent = 'Saving…'; fb.textContent = '';
|
|
const fd = new URLSearchParams();
|
|
fd.append('id', 'fallback');
|
|
fd.append('changes', JSON.stringify([
|
|
{ file: 'master.conf', key: 'FALLBACK_CHECK_INTERVAL', value: String(interval), type: 'scalar' },
|
|
{ file: 'master.conf', key: 'FALLBACK_HANDBACK_STRIKES', value: String(strikes), type: 'scalar' },
|
|
{ file: 'master.conf', key: 'FALLBACK_PARTNERSHIP_SUSPEND_AFTER', value: String(suspend), type: 'scalar' },
|
|
]));
|
|
fetch('/plugins/varaverk/api/confform.php', { method: 'POST', body: fd })
|
|
.then(r => r.json())
|
|
.then(d => {
|
|
btn.disabled = false; btn.textContent = 'Save';
|
|
fb.style.color = d.ok ? '#4caf50' : '#ef5350';
|
|
fb.textContent = d.ok ? 'Saved ✓' : (d.error || 'Failed');
|
|
if (d.ok) setTimeout(() => { fb.textContent = ''; }, 3000);
|
|
})
|
|
.catch(() => {
|
|
btn.disabled = false; btn.textContent = 'Save';
|
|
fb.style.color = '#ef5350'; fb.textContent = 'Request failed';
|
|
});
|
|
};
|
|
|
|
vvFbLoad();
|
|
setInterval(vvFbLoad, 30000);
|
|
|
|
})();
|
|
</script>
|