Rebuild the Fallback tab around what the daemon is actually doing, not what its last state file said
This commit is contained in:
Gmer4Lfe
@@ -0,0 +1,128 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Start and stop the fallback daemon, its dry-run preview, and the failover test harness, on
|
||||
// this host or on a partner. The Fallback tab's only write surface.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Every action shells out to the script that already owns the operation — fallback.sh --stop
|
||||
// and fallback_test.sh --stop — rather than signalling PIDs from PHP. Those two know things
|
||||
// this layer must not have to: which lock holds the process, how long to wait, and, for the
|
||||
// test, that SIGKILL must never be used because only its EXIT trap removes the iptables rule
|
||||
// it installed.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// The scripts own stopping; this endpoint owns routing.
|
||||
// Duplicating the escalation logic here would put a second, divergent implementation of
|
||||
// "how to stop fallback safely" in a language that cannot run its traps.
|
||||
//
|
||||
// Start is dry-run only.
|
||||
// The live daemon is started by array_started.sh at array start, which is the one context
|
||||
// where the tier delays and the state file mean what they say. A button that launched a
|
||||
// live failover monitor mid-session — against a partner mid-maintenance, from a page
|
||||
// someone was reading — is not a button this page should have. Previewing is safe and is
|
||||
// what the page is for.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// POST only. Unraid's CSRF token injector is jQuery-only, so a native fetch() GET would fail
|
||||
// silently anyway; making these POST means a link or prefetch cannot stop a daemon.
|
||||
//
|
||||
// host is matched against the configured host list, never used as a path or a shell word.
|
||||
// The slot resolves to a hostname from conf, then to a Tailscale IP through the same
|
||||
// unambiguous-prefix resolver the rest of the mesh uses. A value that does not name a
|
||||
// configured host is refused before anything runs.
|
||||
//
|
||||
// setsid, not nohup+&, for the dry run.
|
||||
// A backgrounded child stays in php-fpm's process group, and the group kill that ends a
|
||||
// request takes it with it. setsid detaches it into its own session so it survives the
|
||||
// response — the same fix the Scheduler's Stop button needed.
|
||||
//
|
||||
// Output is captured and returned, not discarded.
|
||||
// These scripts report refusals in words — "did not exit within 30s", "NOT force-killing"
|
||||
// — and a boolean would throw away the only explanation the operator gets.
|
||||
//
|
||||
// REQUEST
|
||||
// POST action=stop|stop_test|start_dry|clear_lock host=<slot>
|
||||
//
|
||||
// RESPONSE
|
||||
// {"ok":true,"output":string} action ran; output is the script's own report
|
||||
// {"ok":false,"error":string} bad method, unknown action, or unresolvable host
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/fallback.php vv_pt_peer_lookup(), vv_pt_ts_peers(), vv_pt_ssh()
|
||||
// Fallback/fallback.sh --stop
|
||||
// Fallback/fallback_test.sh --stop
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
require_once dirname(__DIR__) . '/include/fallback.php';
|
||||
|
||||
header('Content-Type: application/json');
|
||||
|
||||
if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
||||
echo json_encode(['ok' => false, 'error' => 'POST only']);
|
||||
exit;
|
||||
}
|
||||
|
||||
$action = (string)($_POST['action'] ?? '');
|
||||
$slot = strtolower((string)($_POST['host'] ?? ''));
|
||||
|
||||
$allowed = ['stop', 'stop_test', 'start_dry', 'clear_lock'];
|
||||
if (!in_array($action, $allowed, true)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'Unknown action']);
|
||||
exit;
|
||||
}
|
||||
|
||||
$hosts = vv_fb_known_hosts();
|
||||
if (!isset($hosts[$slot])) {
|
||||
echo json_encode(['ok' => false, 'error' => 'Unknown host']);
|
||||
exit;
|
||||
}
|
||||
|
||||
$isMe = ($slot === vv_detect_host());
|
||||
$scripts = rtrim(SCRIPTS_DIR, '/');
|
||||
|
||||
// The command, as the script that owns the operation would be invoked by hand.
|
||||
$cmds = [
|
||||
'stop' => 'bash ' . escapeshellarg("$scripts/Fallback/fallback.sh") . ' --stop 2>&1',
|
||||
'stop_test' => 'bash ' . escapeshellarg("$scripts/Fallback/fallback_test.sh") . ' --stop 2>&1',
|
||||
// setsid so it outlives this request; own log so the page can show what the preview said.
|
||||
'start_dry' => 'setsid bash ' . escapeshellarg("$scripts/Fallback/fallback.sh")
|
||||
. ' --dry-run --log > /tmp/varaverk/fallback_dryrun.log 2>&1 < /dev/null & echo started',
|
||||
'clear_lock' => 'rm -f /tmp/unraid_locks/fallback.lock /tmp/unraid_locks/fallback_test.lock && echo cleared',
|
||||
];
|
||||
|
||||
if ($isMe) {
|
||||
@mkdir('/tmp/varaverk', 0755, true);
|
||||
$out = (string)shell_exec($cmds[$action]);
|
||||
echo json_encode(['ok' => true, 'output' => trim($out)]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// Remote: same command, same script, over the SSH this file's neighbours already use.
|
||||
$tsPeers = vv_pt_ts_peers();
|
||||
$ts = vv_pt_peer_lookup($tsPeers, $hosts[$slot]);
|
||||
$ip = $ts['ip'] ?? null;
|
||||
|
||||
$myId = strtoupper(vv_detect_host());
|
||||
$sshKey = vv_fb_scalar(vv_read_conf_raw(vv_detect_host() . '.conf'), $myId . '_SSH_KEY');
|
||||
|
||||
if (!$ip || !$sshKey) {
|
||||
echo json_encode(['ok' => false, 'error' => 'Partner not resolvable — no Tailscale IP or no SSH key']);
|
||||
exit;
|
||||
}
|
||||
|
||||
// The remote's SCRIPTS_DIR is not this host's: appdata mode on one side and flash on the other
|
||||
// is the normal case on this mesh, so ask the partner where it keeps them.
|
||||
$remoteDir = trim((string)vv_pt_ssh($ip, $sshKey,
|
||||
'sed -n \'s/^SCRIPTS_DIR="\(.*\)"$/\1/p\' /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null'));
|
||||
if ($remoteDir === '') $remoteDir = '/boot/config/plugins/varaverk';
|
||||
|
||||
$remoteCmds = [
|
||||
'stop' => "bash '$remoteDir/Fallback/fallback.sh' --stop 2>&1",
|
||||
'stop_test' => "bash '$remoteDir/Fallback/fallback_test.sh' --stop 2>&1",
|
||||
'start_dry' => "mkdir -p /tmp/varaverk; setsid bash '$remoteDir/Fallback/fallback.sh'"
|
||||
. " --dry-run --log > /tmp/varaverk/fallback_dryrun.log 2>&1 < /dev/null & echo started",
|
||||
'clear_lock' => 'rm -f /tmp/unraid_locks/fallback.lock /tmp/unraid_locks/fallback_test.lock && echo cleared',
|
||||
];
|
||||
|
||||
$out = vv_pt_ssh($ip, $sshKey, $remoteCmds[$action]);
|
||||
echo json_encode(['ok' => true, 'output' => trim((string)$out)]);
|
||||
@@ -115,6 +115,63 @@ function vv_fb_remote_running(string $ip, string $sshKey): array {
|
||||
return array_values(array_filter(explode("\n", trim($out))));
|
||||
}
|
||||
|
||||
|
||||
// ── Daemon / process state ────────────────────────────────────────────────────
|
||||
//
|
||||
// The page had no way to say whether fallback.sh was running at all, which is the first thing
|
||||
// anyone looking at this tab wants to know — every state below is written BY that daemon, so a
|
||||
// stale NORMAL from a process that died days ago read exactly like a healthy one.
|
||||
//
|
||||
// Mode matters as much as liveness. A --dry-run instance takes the same `fallback` lock as the
|
||||
// real daemon, so the lock alone cannot tell them apart; the cmdline can, and the per-PID
|
||||
// dry-run state copy is a second confirmation.
|
||||
function vv_fb_proc(string $lockName): array {
|
||||
$lockFile = '/tmp/unraid_locks/' . $lockName . '.lock';
|
||||
$out = ['running' => false, 'pid' => null, 'mode' => null, 'since' => null, 'stale_lock' => false];
|
||||
|
||||
if (!is_file($lockFile)) return $out;
|
||||
|
||||
$pid = (int)strtok((string)@file_get_contents($lockFile), ':');
|
||||
// A lock whose PID is gone is not "running" — it is residue from a SIGKILL or a power cut,
|
||||
// and saying so is the difference between "stop it" and "clear it".
|
||||
if ($pid <= 0 || !is_dir("/proc/$pid")) {
|
||||
$out['stale_lock'] = true;
|
||||
$out['pid'] = $pid ?: null;
|
||||
return $out;
|
||||
}
|
||||
|
||||
$cmd = (string)@file_get_contents("/proc/$pid/cmdline");
|
||||
$args = explode("\0", $cmd);
|
||||
$out['running'] = true;
|
||||
$out['pid'] = $pid;
|
||||
$out['mode'] = in_array('--dry-run', $args, true) || in_array('-n', $args, true) ? 'dry-run' : 'live';
|
||||
$st = @stat("/proc/$pid");
|
||||
if ($st) $out['since'] = (int)$st['mtime'];
|
||||
return $out;
|
||||
}
|
||||
|
||||
// Per-host daemon state. Local reads /proc directly; a partner is asked over the same SSH the
|
||||
// rest of this file already uses, in one call rather than three.
|
||||
function vv_fb_remote_proc(string $ip, string $sshKey): array {
|
||||
$cmd = 'for L in fallback fallback_test; do F=/tmp/unraid_locks/$L.lock; '
|
||||
. 'if [ -f "$F" ]; then P=$(cut -d: -f1 "$F"); '
|
||||
. 'if [ -d "/proc/$P" ]; then M=live; tr "\\0" " " < /proc/$P/cmdline | grep -q -- "--dry-run" && M=dry-run; '
|
||||
. 'echo "$L:running:$P:$M"; else echo "$L:stale:$P:"; fi; else echo "$L:none::"; fi; done';
|
||||
$out = vv_pt_ssh($ip, $sshKey, $cmd);
|
||||
$res = ['fallback' => ['running' => false, 'pid' => null, 'mode' => null, 'stale_lock' => false],
|
||||
'fallback_test' => ['running' => false, 'pid' => null, 'mode' => null, 'stale_lock' => false]];
|
||||
foreach (explode("\n", trim((string)$out)) as $line) {
|
||||
$p = explode(':', trim($line));
|
||||
if (count($p) < 4 || !isset($res[$p[0]])) continue;
|
||||
if ($p[1] === 'running') {
|
||||
$res[$p[0]] = ['running' => true, 'pid' => (int)$p[2], 'mode' => $p[3] ?: 'live', 'stale_lock' => false];
|
||||
} elseif ($p[1] === 'stale') {
|
||||
$res[$p[0]]['stale_lock'] = true;
|
||||
$res[$p[0]]['pid'] = (int)$p[2] ?: null;
|
||||
}
|
||||
}
|
||||
return $res;
|
||||
}
|
||||
// ── Covers — what a node runs for the other when it's down ───────────────────
|
||||
|
||||
function vv_fb_covers(string $covering, string $remote, string $coveringRaw, string $remoteRaw): array {
|
||||
@@ -207,14 +264,51 @@ function vv_fb_all(): array {
|
||||
break; // 2-node only
|
||||
}
|
||||
|
||||
// Daemon liveness per node. Everything in $state was written by this process — without
|
||||
// it a NORMAL left behind by a daemon that died days ago is indistinguishable from a
|
||||
// NORMAL being refreshed every 30 seconds.
|
||||
if ($isMe) {
|
||||
$proc = vv_fb_proc('fallback');
|
||||
$procTest = vv_fb_proc('fallback_test');
|
||||
} elseif ($ip && $mySshKey && $ts['online']) {
|
||||
$rp = vv_fb_remote_proc($ip, $mySshKey);
|
||||
$proc = $rp['fallback'];
|
||||
$procTest = $rp['fallback_test'];
|
||||
} else {
|
||||
$proc = ['running' => null, 'pid' => null, 'mode' => null, 'stale_lock' => false];
|
||||
$procTest = ['running' => null, 'pid' => null, 'mode' => null, 'stale_lock' => false];
|
||||
}
|
||||
|
||||
// How fresh the state actually is. The daemon rewrites its file every check interval, so
|
||||
// an age far past that interval means it is wedged even while the process still exists.
|
||||
$stateAge = null;
|
||||
if ($isMe) {
|
||||
$sp = STATE_DIR . '/fallback_state.db';
|
||||
if (is_file($sp)) $stateAge = time() - (int)@filemtime($sp);
|
||||
}
|
||||
|
||||
// "Reachable" is three separate facts and one boolean hid which had failed.
|
||||
$reach = [
|
||||
'tailscale' => $isMe ? true : ($ts['online'] === true),
|
||||
'ip' => $isMe ? null : $ip,
|
||||
'ssh' => $isMe ? true : ($running !== [] || ($proc['running'] !== null)),
|
||||
'state_file' => ($state['state'] ?? 'UNKNOWN') !== 'UNKNOWN',
|
||||
];
|
||||
|
||||
$nodes[] = [
|
||||
'slot' => $slot,
|
||||
'id' => strtoupper($slot),
|
||||
'hostname' => $hostname,
|
||||
'is_me' => $isMe,
|
||||
'ts_online' => $ts['online'],
|
||||
'ts_ip' => $ip,
|
||||
'state' => $state,
|
||||
'state_age' => $stateAge,
|
||||
'running' => $running,
|
||||
'running_count' => count($running),
|
||||
'proc' => $proc,
|
||||
'proc_test' => $procTest,
|
||||
'reach' => $reach,
|
||||
'covers' => $covers,
|
||||
];
|
||||
}
|
||||
|
||||
@@ -16,22 +16,81 @@
|
||||
// outage, when stale numbers are actively misleading.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Read-only. The page cannot trigger a failover, force a handback, or start a covered
|
||||
// container. Fallback is driven by fallback.sh reacting to real reachability, and a manual
|
||||
// override from a browser is exactly the wrong way to enter that state.
|
||||
// Cannot cause a failover. The page stops daemons, starts a PREVIEW, and edits conf — it has
|
||||
// no control that enters FALLBACK or forces a handback. Fallback is driven by fallback.sh
|
||||
// reacting to real reachability, and a manual override from a browser is exactly the wrong
|
||||
// way to enter that state. The one start control is --dry-run, which changes nothing.
|
||||
//
|
||||
// A missing state file renders as unknown, never as NORMAL — claiming healthy for a
|
||||
// fallback process that is not running would be the worst possible error on this page.
|
||||
//
|
||||
// State age is shown next to state, always. Every value in the state file was written by a
|
||||
// daemon that may not be running: this host displayed NORMAL from a file five days stale
|
||||
// with no process alive, and nothing on the page said so.
|
||||
//
|
||||
// Stopping the test is a different button from stopping the daemon, deliberately. The test
|
||||
// holds an iptables rule that only its own EXIT trap removes, so the two cannot share a
|
||||
// control that might escalate to SIGKILL.
|
||||
//
|
||||
// RENDERS
|
||||
// Per-node state, tier activation and delays, handback strikes, covered container status
|
||||
// Per-node state and freshness, daemon liveness and mode, reachability legs, tier activation
|
||||
// and delays, handback strikes, covered container status, quick settings, assistant
|
||||
//
|
||||
// DEPENDS ON
|
||||
// api/fallback.php polled every 30s → include/fallback.php
|
||||
// api/fallback_control.php start/stop actions → Fallback/fallback*.sh
|
||||
// api/confform.php inline conf edits → include/confform.php
|
||||
require_once dirname(__DIR__) . '/include/confui.php';
|
||||
require_once dirname(__DIR__) . '/include/ai_chat.php';
|
||||
|
||||
// Same shared surface as the Partnership and Monitor tabs, gated the same way. Fallback
|
||||
// questions — why a tier has not fired, what the mirror would actually start, whether a stale
|
||||
// state file matters — are asked while looking at this page.
|
||||
if (vv_ai_ui_on()) vv_ai_chat_assets();
|
||||
?>
|
||||
<style>
|
||||
/* ── Host card: daemon, freshness, reachability, controls ── */
|
||||
/* Class names are all vv-fb-* prefixed. Unraid Connect injects a global Tailwind layer into
|
||||
every page, so a bare utility-shaped name like `fixed` or `grid` would be captured by it. */
|
||||
.vv-fb-hcard { background:#141414;border:1px solid #262626;border-radius:6px;padding:12px 13px;min-width:0; }
|
||||
.vv-fb-hcard.me { border-color:#2a3a2a; }
|
||||
.vv-fb-hcard.warn { border-color:#4a3800; }
|
||||
.vv-fb-hrow { display:flex;align-items:center;gap:7px;margin-bottom:8px;flex-wrap:wrap; }
|
||||
.vv-fb-hid { font-size:13px;font-weight:700;color:#ddd; }
|
||||
.vv-fb-hnm { font-size:11px;color:#666;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;min-width:0; }
|
||||
.vv-fb-usbadge { background:#1a3a1a;color:#4caf50;font-size:8px;padding:1px 5px;border-radius:3px; }
|
||||
|
||||
.vv-fb-stats { display:grid;grid-template-columns:auto 1fr;gap:3px 10px;font-size:11px;margin-bottom:9px; }
|
||||
.vv-fb-stats b { color:#444;font-weight:400; }
|
||||
.vv-fb-sv { color:#999;min-width:0; }
|
||||
.vv-fb-sv.good { color:#4caf50; }
|
||||
.vv-fb-sv.warn { color:#ffb74d; }
|
||||
.vv-fb-sv.bad { color:#ef5350; }
|
||||
.vv-fb-sv.dim { color:#3a3a3a; }
|
||||
|
||||
/* Reachability legs — separate facts, never rolled into one boolean. */
|
||||
.vv-fb-legs { display:flex;gap:5px;flex-wrap:wrap;margin-bottom:9px; }
|
||||
.vv-fb-leg { font-size:9px;padding:1px 6px;border-radius:3px;border:1px solid #222;background:#111;color:#444; }
|
||||
.vv-fb-leg.ok { border-color:#243a24;background:#101a10;color:#4caf50; }
|
||||
.vv-fb-leg.no { border-color:#3a1e1e;background:#1a1010;color:#ef5350; }
|
||||
|
||||
.vv-fb-acts { display:flex;gap:6px;flex-wrap:wrap;margin-top:9px;padding-top:8px;border-top:1px solid #1e1e1e; }
|
||||
.vv-fb-btn { background:#111;border:1px solid #262626;color:#888;border-radius:3px;
|
||||
padding:4px 9px;cursor:pointer;font-size:10px;white-space:nowrap; }
|
||||
.vv-fb-btn:hover:not(:disabled) { border-color:#3a3a3a;color:#ccc; }
|
||||
.vv-fb-btn:disabled { opacity:.4;cursor:default; }
|
||||
.vv-fb-btn.stop { color:#ef9a9a; }
|
||||
.vv-fb-btn.go { color:#4a9eff; }
|
||||
.vv-fb-out { font-size:9px;color:#555;margin-top:6px;white-space:pre-wrap;word-break:break-word;
|
||||
max-height:80px;overflow:auto;font-family:monospace; }
|
||||
|
||||
/* Quick settings — one card per setting rather than one card of rows, so each carries its own
|
||||
why. The old single card listed three numbers with no indication what changing one costs. */
|
||||
.vv-fb-qs { display:grid;grid-template-columns:repeat(auto-fit,minmax(190px,1fr));gap:10px;margin-bottom:12px; }
|
||||
.vv-fb-qcard { background:#141414;border:1px solid #242424;border-radius:6px;padding:10px 11px; }
|
||||
.vv-fb-qlbl { font-size:11px;color:#aaa;font-weight:600;margin-bottom:2px; }
|
||||
.vv-fb-qsub { font-size:9px;color:#3f3f3f;line-height:1.35;margin-bottom:7px; }
|
||||
.vv-fb-qin { display:flex;align-items:center;gap:7px; }
|
||||
/* ── Existing status styles ── */
|
||||
.vv-fb-active { background:#1a1200;border:1px solid #5a3800;border-radius:6px;padding:12px 14px; }
|
||||
.vv-fb-active-h { display:flex;align-items:baseline;gap:10px;margin-bottom:8px; }
|
||||
@@ -125,50 +184,77 @@ require_once dirname(__DIR__) . '/include/confui.php';
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Mesh verdict — one line answering "is this armed and is anything watching" before any detail -->
|
||||
<div class="vv-fb-card" id="vv-fb-verdict-card" style="margin-bottom:12px;">
|
||||
<div id="vv-fb-verdict" style="font-size:12px;color:#555;">Loading…</div>
|
||||
</div>
|
||||
|
||||
<!-- Status section -->
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;padding:0 2px;">
|
||||
<span style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;">Status</span>
|
||||
<span style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;">Nodes</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;" id="vv-fb-ts"></span>
|
||||
</div>
|
||||
|
||||
<div id="vv-fb-grid" style="display:grid;grid-template-columns:repeat(8,1fr);gap:12px;margin-bottom:14px;">
|
||||
<div id="vv-fb-grid" style="display:grid;grid-template-columns:repeat(auto-fit,minmax(340px,1fr));gap:12px;margin-bottom:14px;">
|
||||
<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">Loading…</div>
|
||||
</div>
|
||||
|
||||
<!-- Settings card -->
|
||||
<div class="vv-fb-card">
|
||||
<div class="vv-fb-card-hdr">Settings</div>
|
||||
<!-- Assistant + mesh chat -->
|
||||
<?php if (vv_ai_ui_on()): ?>
|
||||
<div class="vv-card" id="vv-fb-ai-card" style="margin-bottom:12px;">
|
||||
<?php vv_ai_chat_markup('vv-fb-ai', [
|
||||
'profile' => 'varaverk',
|
||||
'compact' => true,
|
||||
// Fallback is the mesh's business by definition — a failover is two hosts agreeing about
|
||||
// each other — so the mesh side leads here, as it does on Partnership.
|
||||
'mesh' => true,
|
||||
'meshDefault' => true,
|
||||
'title' => 'Assistant',
|
||||
'height' => '300px',
|
||||
'tall' => '500px',
|
||||
'tallLarge' => '750px',
|
||||
'empty' => 'Ask about fallback — why a tier has not fired, what the partner would '
|
||||
. 'start if this host went dark, whether a stale state file matters.',
|
||||
]); ?>
|
||||
</div>
|
||||
<?php endif; ?>
|
||||
|
||||
<div class="vv-fb-set-row">
|
||||
<span class="vv-fb-set-lbl">Check interval</span>
|
||||
<div style="display:flex;align-items:center;gap:8px;">
|
||||
<!-- Quick settings -->
|
||||
<div style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;margin-bottom:10px;padding:0 2px;">Quick settings</div>
|
||||
|
||||
<div class="vv-fb-qs">
|
||||
<div class="vv-fb-qcard">
|
||||
<div class="vv-fb-qlbl">Check interval</div>
|
||||
<div class="vv-fb-qsub">How often the daemon re-tests the partner. Also the age at which a state file is stale.</div>
|
||||
<div class="vv-fb-qin">
|
||||
<input class="vv-fb-set-inp" id="vv-fb-interval" type="number" min="5" max="300">
|
||||
<span class="vv-fb-set-unit">seconds</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="vv-fb-set-row">
|
||||
<span class="vv-fb-set-lbl">Handback strikes</span>
|
||||
<div style="display:flex;align-items:center;gap:8px;">
|
||||
<div class="vv-fb-qcard">
|
||||
<div class="vv-fb-qlbl">Handback strikes</div>
|
||||
<div class="vv-fb-qsub">Consecutive healthy checks before handing services back. Higher rides out a flapping link.</div>
|
||||
<div class="vv-fb-qin">
|
||||
<input class="vv-fb-set-inp" id="vv-fb-strikes" type="number" min="1" max="20">
|
||||
<span class="vv-fb-set-unit">consecutive</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="vv-fb-set-row">
|
||||
<span class="vv-fb-set-lbl">Partnership suspend after</span>
|
||||
<div style="display:flex;align-items:center;gap:8px;">
|
||||
<div class="vv-fb-qcard">
|
||||
<div class="vv-fb-qlbl">Partnership suspend after</div>
|
||||
<div class="vv-fb-qsub">Minutes without an active partnership before fallback suspends itself. 0 suspends immediately.</div>
|
||||
<div class="vv-fb-qin">
|
||||
<input class="vv-fb-set-inp" id="vv-fb-suspend" type="number" min="0" max="1440">
|
||||
<span class="vv-fb-set-unit">minutes</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style="display:flex;justify-content:flex-end;align-items:center;gap:10px;margin-top:10px;padding-top:8px;border-top:1px solid #1e1e1e;">
|
||||
<span id="vv-fb-set-fb" style="font-size:11px;"></span>
|
||||
<button class="vv-fb-save-btn" id="vv-fb-save-btn" onclick="vvFbSaveSettings()">Save</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style="display:flex;justify-content:flex-end;align-items:center;gap:10px;margin-bottom:14px;">
|
||||
<span id="vv-fb-set-fb" style="font-size:11px;"></span>
|
||||
<button class="vv-fb-save-btn" id="vv-fb-save-btn" onclick="vvFbSaveSettings()">Save</button>
|
||||
</div>
|
||||
|
||||
<?php
|
||||
// The conf sections this page is about, drawn by the shared renderer. They were reachable
|
||||
@@ -303,6 +389,9 @@ function _tierSection(tiers, activeTier, delays) {
|
||||
}).join('');
|
||||
}
|
||||
|
||||
// Only rendered against FRESH state — see _nodeCard. A grace timer counted off a state file
|
||||
// nobody has written for five days reads as an active grace window, which is precisely the kind
|
||||
// of stale-as-current claim this page must not make.
|
||||
function _ptStatus(st) {
|
||||
if (!st) return '';
|
||||
if (st.partnership_suspended) return _stateBadge('SUSPENDED');
|
||||
@@ -313,37 +402,153 @@ function _ptStatus(st) {
|
||||
return '';
|
||||
}
|
||||
|
||||
function _nodeCard(node, suspendAfter) {
|
||||
function _age(sec) {
|
||||
if (sec === null || sec === undefined) return '—';
|
||||
if (sec < 60) return sec + 's ago';
|
||||
if (sec < 3600) return Math.floor(sec / 60) + 'm ago';
|
||||
if (sec < 86400) return Math.floor(sec / 3600) + 'h ago';
|
||||
return Math.floor(sec / 86400) + 'd ago';
|
||||
}
|
||||
|
||||
// Daemon liveness, stated as one of four distinct things rather than a boolean. "Not running"
|
||||
// and "running a preview" are opposite operational facts and the old page showed neither.
|
||||
function _daemonRow(p, checkInterval) {
|
||||
if (!p) return ['dim', 'unknown', ''];
|
||||
if (p.stale_lock) return ['bad', 'not running · stale lock' + (p.pid ? ' (PID ' + p.pid + ')' : ''), 'stale'];
|
||||
if (p.running === null) return ['dim', 'unknown — partner not reachable', ''];
|
||||
if (!p.running) return ['bad', 'not running', ''];
|
||||
const mode = p.mode === 'dry-run'
|
||||
? '<span style="color:#4a9eff;">dry run</span>'
|
||||
: '<span style="color:#4caf50;">live</span>';
|
||||
return ['good', mode + ' · PID ' + p.pid, 'up'];
|
||||
}
|
||||
|
||||
// State freshness. The daemon rewrites this file every check interval, so anything past a few
|
||||
// intervals is a wedged process or a dead one — and the state value it left behind is a claim
|
||||
// about the past, not the present.
|
||||
function _freshness(node, checkInterval) {
|
||||
const a = node.state_age;
|
||||
if (a === null || a === undefined) {
|
||||
return node.is_me
|
||||
? ['bad', 'no state file — fallback has never run here']
|
||||
: ['dim', 'not readable from here'];
|
||||
}
|
||||
const limit = Math.max(120, (checkInterval || 30) * 4);
|
||||
return [a > limit ? 'bad' : 'good', _age(a) + (a > limit ? ' — stale' : '')];
|
||||
}
|
||||
|
||||
function _leg(ok, label) {
|
||||
const cls = ok === true ? 'ok' : ok === false ? 'no' : '';
|
||||
const sym = ok === true ? '✓' : ok === false ? '✕' : '·';
|
||||
return `<span class="vv-fb-leg ${cls}">${sym} ${label}</span>`;
|
||||
}
|
||||
|
||||
function _nodeCard(node, data) {
|
||||
const st = node.state || {};
|
||||
const state = st.partnership_suspended ? 'SUSPENDED' : (st.state || 'UNKNOWN');
|
||||
const cov = node.covers;
|
||||
const active = _activeTier(state === 'FALLBACK' ? st : null);
|
||||
const slot = node.slot;
|
||||
|
||||
const [dCls, dTxt, dKind] = _daemonRow(node.proc, data.check_interval);
|
||||
const [fCls, fTxt] = _freshness(node, data.check_interval);
|
||||
const test = node.proc_test || {};
|
||||
const reach = node.reach || {};
|
||||
|
||||
const covTarget = cov
|
||||
? `<span class="vv-fb-arrow">→</span><span class="vv-fb-covers">covers ${cov.id} (${cov.hostname})</span>`
|
||||
: '';
|
||||
? `<span class="vv-fb-sv">${cov.id} <span style="color:#3a3a3a;">(${vvEscHtml(cov.hostname)})</span></span>`
|
||||
: '<span class="vv-fb-sv dim">nothing configured</span>';
|
||||
|
||||
const tierSection = cov
|
||||
? _tierSection(cov, active, cov.delays)
|
||||
: '<div style="color:#3a3a3a;font-size:11px;">No coverage configured</div>';
|
||||
|
||||
const ptBadge = node.is_me ? _ptStatus(st) : '';
|
||||
// Tier lists live in the COVERED host's conf, so an empty set here is a config gap on the
|
||||
// other side, not on this one. Saying which conf to edit saves the hunt.
|
||||
const tierTotal = cov ? ['tier1','tier2','tier3','tier4'].reduce((n,k)=>n+((cov[k]||[]).length),0) : 0;
|
||||
const tierWarn = (cov && tierTotal === 0)
|
||||
? `<div style="font-size:9px;color:#5a4020;margin-top:5px;">No containers in any tier — set FALLBACK_${cov.id}_TIER1 in ${cov.slot}.conf</div>`
|
||||
: '';
|
||||
|
||||
return `<div class="vv-card vv-fb-node">
|
||||
<div class="vv-fb-node-h">
|
||||
const busy = _vvFbBusy[slot] ? 'disabled' : '';
|
||||
const acts = `
|
||||
<button class="vv-fb-btn stop" ${busy} onclick="vvFbAct('${slot}','stop',this)"
|
||||
title="fallback.sh --stop — SIGTERM, then SIGKILL after 10s">Stop fallback</button>
|
||||
<button class="vv-fb-btn go" ${busy} onclick="vvFbAct('${slot}','start_dry',this)"
|
||||
title="fallback.sh --dry-run — previews decisions, changes nothing">Start dry run</button>
|
||||
<button class="vv-fb-btn stop" ${busy} onclick="vvFbAct('${slot}','stop_test',this)"
|
||||
title="fallback_test.sh --stop — SIGTERM only, never SIGKILL: only its trap removes the iptables rule">Stop test</button>
|
||||
${(node.proc?.stale_lock || test.stale_lock)
|
||||
? `<button class="vv-fb-btn" ${busy} onclick="vvFbAct('${slot}','clear_lock',this)"
|
||||
title="Remove lock files left by a killed run">Clear lock</button>` : ''}`;
|
||||
|
||||
const testRow = test.running
|
||||
? `<b>Test</b><span class="vv-fb-sv warn">running · PID ${test.pid} — holds an iptables rule</span>`
|
||||
: test.stale_lock
|
||||
? `<b>Test</b><span class="vv-fb-sv bad">stale lock — an iptables rule may be stranded</span>`
|
||||
: '';
|
||||
|
||||
return `<div class="vv-fb-hcard${node.is_me ? ' me' : ''}${(dCls === 'bad' || fCls === 'bad') ? ' warn' : ''}">
|
||||
<div class="vv-fb-hrow">
|
||||
${_stateDot(state)}
|
||||
<span class="vv-fb-node-id">${node.id}</span>
|
||||
<span class="vv-fb-node-nm">${node.hostname}</span>
|
||||
${covTarget}
|
||||
<span class="vv-fb-hid">${node.id}</span>
|
||||
<span class="vv-fb-hnm">${vvEscHtml(node.hostname)}</span>
|
||||
${node.is_me ? '<span class="vv-fb-usbadge">US</span>' : ''}
|
||||
<span style="flex:1"></span>
|
||||
${ptBadge}
|
||||
${(node.is_me && fCls !== 'bad') ? _ptStatus(st) : ''}
|
||||
${_stateBadge(state)}
|
||||
</div>
|
||||
|
||||
<div class="vv-fb-legs">
|
||||
${_leg(reach.tailscale, 'tailscale')}
|
||||
${_leg(node.is_me ? true : reach.ssh, 'ssh')}
|
||||
${_leg(reach.state_file, 'state file')}
|
||||
${node.ts_ip ? `<span class="vv-fb-leg">${vvEscHtml(node.ts_ip)}</span>` : ''}
|
||||
</div>
|
||||
|
||||
<div class="vv-fb-stats">
|
||||
<b>Daemon</b> <span class="vv-fb-sv ${dCls}">${dTxt}</span>
|
||||
<b>State age</b> <span class="vv-fb-sv ${fCls}">${fTxt}</span>
|
||||
<b>Covers</b> ${covTarget}
|
||||
<b>Running</b> <span class="vv-fb-sv">${node.running_count ?? 0} containers</span>
|
||||
${state === 'FALLBACK'
|
||||
? `<b>Strikes</b><span class="vv-fb-sv warn">${st.handback_strikes} / ${data.handback_req || 3}</span>` : ''}
|
||||
${testRow}
|
||||
</div>
|
||||
|
||||
<hr class="vv-fb-sep">
|
||||
${tierSection}
|
||||
${tierWarn}
|
||||
|
||||
<div class="vv-fb-acts">${acts}</div>
|
||||
<div class="vv-fb-out" id="vv-fb-out-${slot}"></div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// One mesh-level sentence, above the detail. Answers the two questions the page exists for
|
||||
// before anyone has to read a card: is fallback armed, and is anything actually watching.
|
||||
function _verdict(data) {
|
||||
const el = document.getElementById('vv-fb-verdict');
|
||||
if (!el) return;
|
||||
const nodes = data.nodes || [];
|
||||
if (!data.partnership_enabled)
|
||||
return void (el.innerHTML = '<span style="color:#ef5350;">PARTNERSHIP_ENABLED=false</span> — all cross-server operations are disabled.');
|
||||
if (!data.fb_enabled)
|
||||
return void (el.innerHTML = '<span style="color:#ffb74d;">FALLBACK_ENABLED=false</span> — nothing will fail over. Arming it takes effect at the next array start.');
|
||||
|
||||
const inFb = nodes.filter(n => (n.state?.state) === 'FALLBACK');
|
||||
const live = nodes.filter(n => n.proc?.running && n.proc.mode === 'live');
|
||||
const dry = nodes.filter(n => n.proc?.running && n.proc.mode === 'dry-run');
|
||||
const dead = nodes.filter(n => n.proc && n.proc.running === false);
|
||||
|
||||
if (inFb.length)
|
||||
return void (el.innerHTML = `<span style="color:#ffb74d;font-weight:600;">${inFb.map(n=>n.id).join(', ')} in FALLBACK</span> — covering for a partner right now.`);
|
||||
|
||||
let s = `<span style="color:#4caf50;">Armed</span> · ${live.length}/${nodes.length} node${nodes.length!==1?'s':''} running the daemon`;
|
||||
if (dry.length) s += ` · <span style="color:#4a9eff;">${dry.length} in dry run</span>`;
|
||||
if (dead.length) s += ` · <span style="color:#ef5350;">${dead.map(n=>n.id).join(', ')} not running — nothing would detect an outage there</span>`;
|
||||
el.innerHTML = s;
|
||||
}
|
||||
function _setToggles(data) {
|
||||
const pairs = [
|
||||
['vv-fb-pt-tog', !!data.partnership_enabled],
|
||||
@@ -369,8 +574,10 @@ function _setInputs(data) {
|
||||
}
|
||||
|
||||
function _render(data) {
|
||||
_vvFbLast = data;
|
||||
_setToggles(data);
|
||||
_setInputs(data);
|
||||
_verdict(data);
|
||||
|
||||
const grid = document.getElementById('vv-fb-grid');
|
||||
|
||||
@@ -381,7 +588,7 @@ function _render(data) {
|
||||
} else {
|
||||
const nodes = data.nodes || [];
|
||||
let html = _activeCard(nodes, data.handback_req || 3);
|
||||
for (const node of nodes) html += _nodeCard(node, data.suspend_after || 120);
|
||||
for (const node of nodes) html += _nodeCard(node, data);
|
||||
grid.innerHTML = html || '<div class="vv-fb-disabled">No nodes configured.</div>';
|
||||
}
|
||||
|
||||
@@ -403,6 +610,59 @@ function vvFbLoad() {
|
||||
});
|
||||
}
|
||||
|
||||
// Per-slot in-flight guard, so a second click cannot fire while a stop is still escalating.
|
||||
// Keyed by slot rather than one page-wide flag: acting on one host must not freeze the other's
|
||||
// controls, and on this page the two hosts are routinely in different states.
|
||||
const _vvFbBusy = {};
|
||||
let _vvFbLast = null;
|
||||
|
||||
// URLSearchParams, not FormData. Unraid's CSRF token injector is jQuery-only, and a multipart
|
||||
// body from native fetch() hangs here with no status and no server-side trace.
|
||||
window.vvFbAct = async function (slot, action, btn) {
|
||||
if (_vvFbBusy[slot]) return;
|
||||
|
||||
const labels = {
|
||||
stop: 'Stop the fallback daemon on ' + slot.toUpperCase() + '?',
|
||||
stop_test: 'Stop the fallback TEST on ' + slot.toUpperCase() + '?\n\nSIGTERM only — the test needs its own trap to remove the iptables rule it installed.',
|
||||
start_dry: 'Start a fallback DRY RUN on ' + slot.toUpperCase() + '?\n\nIt previews decisions and changes nothing.',
|
||||
clear_lock: 'Clear fallback lock files on ' + slot.toUpperCase() + '?\n\nOnly do this when no fallback process is running.',
|
||||
};
|
||||
|
||||
// vvConfirm, never native confirm() — a native dialog offers "prevent additional dialogs",
|
||||
// which kills every later dialog on the page document-wide. It returns a PROMISE, not a
|
||||
// callback: passing a function would land in its opts argument and the body would never run,
|
||||
// leaving a button that silently does nothing.
|
||||
if (!await vvConfirm(labels[action] || ('Run ' + action + '?'))) return;
|
||||
|
||||
_vvFbBusy[slot] = true;
|
||||
const out = document.getElementById('vv-fb-out-' + slot);
|
||||
if (out) { out.style.color = '#555'; out.textContent = 'Running ' + action + '…'; }
|
||||
document.querySelectorAll('.vv-fb-btn').forEach(b => { b.disabled = true; });
|
||||
|
||||
const fd = new URLSearchParams();
|
||||
fd.append('action', action);
|
||||
fd.append('host', slot);
|
||||
|
||||
fetch('/plugins/varaverk/api/fallback_control.php', { method: 'POST', body: fd })
|
||||
.then(r => r.json())
|
||||
.then(d => {
|
||||
if (out) {
|
||||
// The scripts explain refusals in words — "did not exit within 30s", "NOT
|
||||
// force-killing" — so the reply is shown verbatim rather than reduced to ok/failed.
|
||||
out.style.color = d.ok ? '#555' : '#ef5350';
|
||||
out.textContent = d.ok ? (d.output || 'done') : (d.error || 'failed');
|
||||
}
|
||||
})
|
||||
.catch(e => { if (out) { out.style.color = '#ef5350'; out.textContent = 'Request failed: ' + e; } })
|
||||
.finally(() => {
|
||||
_vvFbBusy[slot] = false;
|
||||
document.querySelectorAll('.vv-fb-btn').forEach(b => { b.disabled = false; });
|
||||
// Re-poll rather than guessing: whether the daemon actually stopped is a fact to read
|
||||
// back, not one to infer from the request having returned.
|
||||
setTimeout(vvFbLoad, 1200);
|
||||
});
|
||||
};
|
||||
|
||||
window.vvFbToggle = function(track, key) {
|
||||
const on = !track.classList.contains('on');
|
||||
track.classList.toggle('on', on);
|
||||
|
||||
Reference in New Issue
Block a user