372 lines
18 KiB
PHP
372 lines
18 KiB
PHP
<?php
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
// PURPOSE
|
|
// Fallback page data layer. Reports what state each node is in (NORMAL / FALLBACK /
|
|
// NO_INTERNET / DARK), which tiers have activated, and which covered containers are
|
|
// actually running — for this host and for every partner.
|
|
//
|
|
// DESIGN PRINCIPLES
|
|
// Observes fallback.sh; never participates in it.
|
|
// State is read from the state file the running fallback process maintains. This file
|
|
// has no opinion about whether a failover should happen and cannot trigger, advance,
|
|
// or hand back one. The page is a window, not a lever.
|
|
//
|
|
// Remote state is read the same way local state is.
|
|
// vv_fb_remote_state() SSHes over and reads the partner's own fallback_state.db rather
|
|
// than inferring the partner's state from what this host can see. A node's state is
|
|
// whatever that node believes, not what its neighbour guesses.
|
|
//
|
|
// Coverage lists come from conf, tier membership from FALLBACK_<HOST>_TIER<n>.
|
|
// Named for the host being covered, not the host doing the covering — see the fallback
|
|
// section of the top-level README for why that reads backwards at first.
|
|
//
|
|
// OPERATIONAL SAFEGUARDS
|
|
// A missing state file parses as empty, not as NORMAL.
|
|
// vv_fb_local_state() hands an empty string to the parser when the file is absent, so
|
|
// the page shows unknown rather than asserting everything is fine. Reporting a healthy
|
|
// state for a fallback process that is not running would be the worst possible lie on
|
|
// this page.
|
|
//
|
|
// An unreachable partner degrades to what is locally known.
|
|
// Remote SSH failures return empty rather than propagating an error, so one dark node
|
|
// cannot blank the whole page — which is precisely the situation this page exists for.
|
|
//
|
|
// Read-only over SSH.
|
|
// The only remote commands issued are a state-file read and `docker ps`. Nothing here
|
|
// starts or stops a container on either side.
|
|
//
|
|
// EXPORTS
|
|
// State vv_fb_local_state(), vv_fb_remote_state(), vv_fb_parse_state()
|
|
// Containers vv_fb_local_running(), vv_fb_remote_running(), vv_fb_covers()
|
|
// Assembly vv_fb_known_hosts(), vv_fb_all()
|
|
// Parsing vv_fb_bash_array(), vv_fb_scalar()
|
|
//
|
|
// CONFIGURATION
|
|
// STATE_DIR fallback_state.db — written by Fallback/fallback.sh
|
|
// FALLBACK_<HOST>_TIER1..4 containers covered per tier, keyed by the covered host
|
|
// HOST*_SSH_KEY used to read partner state
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
require_once __DIR__ . '/config.php';
|
|
require_once __DIR__ . '/partnership.php'; // vv_pt_ssh(), vv_pt_ts_peers()
|
|
|
|
// ── Conf parsers ──────────────────────────────────────────────────────────────
|
|
|
|
function vv_fb_bash_array(string $raw, string $varname): array {
|
|
return vv_parse_bash_array($raw, $varname);
|
|
}
|
|
|
|
function vv_fb_scalar(string $raw, string $varname): string {
|
|
return vv_parse_conf_scalar($raw, $varname);
|
|
}
|
|
|
|
// ── State file ────────────────────────────────────────────────────────────────
|
|
|
|
function vv_fb_parse_state(string $text): array {
|
|
$out = [
|
|
'state' => 'UNKNOWN',
|
|
'fallback_start' => 0,
|
|
'handback_strikes' => 0,
|
|
'tier2_started' => false,
|
|
'tier3_started' => false,
|
|
'tier4_started' => false,
|
|
'partnership_suspended' => false,
|
|
'partner_lost_at' => 0,
|
|
];
|
|
foreach (explode("\n", $text) as $line) {
|
|
$line = trim($line);
|
|
if (!$line || !str_contains($line, '=')) continue;
|
|
[$k, $v] = array_pad(explode('=', $line, 2), 2, '');
|
|
$k = trim($k); $v = trim($v, '"\'');
|
|
switch ($k) {
|
|
case 'state': $out['state'] = $v; break;
|
|
case 'fallback_start': $out['fallback_start'] = (int)$v; break;
|
|
case 'handback_strikes': $out['handback_strikes'] = (int)$v; break;
|
|
case 'tier2_started': $out['tier2_started'] = $v === 'true'; break;
|
|
case 'tier3_started': $out['tier3_started'] = $v === 'true'; break;
|
|
case 'tier4_started': $out['tier4_started'] = $v === 'true'; break;
|
|
case 'partnership_suspended': $out['partnership_suspended'] = $v === 'true'; break;
|
|
case 'partner_lost_at': $out['partner_lost_at'] = (int)$v; break;
|
|
}
|
|
}
|
|
return $out;
|
|
}
|
|
|
|
function vv_fb_local_state(): array {
|
|
$path = STATE_DIR . '/fallback_state.db';
|
|
return vv_fb_parse_state(file_exists($path) ? file_get_contents($path) : '');
|
|
}
|
|
|
|
function vv_fb_remote_state(string $ip, string $sshKey): array {
|
|
$out = vv_pt_ssh($ip, $sshKey, vv_remote_state_cmd('fallback_state.db'));
|
|
return vv_fb_parse_state($out);
|
|
}
|
|
|
|
// ── Running containers ────────────────────────────────────────────────────────
|
|
|
|
function vv_fb_local_running(): array {
|
|
$out = shell_exec("docker ps --format '{{.Names}}' 2>/dev/null") ?: '';
|
|
return array_values(array_filter(explode("\n", trim($out))));
|
|
}
|
|
|
|
function vv_fb_remote_running(string $ip, string $sshKey): array {
|
|
$out = vv_pt_ssh($ip, $sshKey, "docker ps --format '{{.Names}}' 2>/dev/null");
|
|
return array_values(array_filter(explode("\n", trim($out))));
|
|
}
|
|
|
|
|
|
// ── Daemon / process state ────────────────────────────────────────────────────
|
|
//
|
|
// The page had no way to say whether fallback.sh was running at all, which is the first thing
|
|
// anyone looking at this tab wants to know — every state below is written BY that daemon, so a
|
|
// stale NORMAL from a process that died days ago read exactly like a healthy one.
|
|
//
|
|
// Mode matters as much as liveness. A --dry-run instance takes the same `fallback` lock as the
|
|
// real daemon, so the lock alone cannot tell them apart; the cmdline can, and the per-PID
|
|
// dry-run state copy is a second confirmation.
|
|
function vv_fb_proc(string $lockName): array {
|
|
$lockFile = '/tmp/unraid_locks/' . $lockName . '.lock';
|
|
$out = ['running' => false, 'pid' => null, 'mode' => null, 'since' => null, 'stale_lock' => false];
|
|
|
|
if (!is_file($lockFile)) return $out;
|
|
|
|
$pid = (int)strtok((string)@file_get_contents($lockFile), ':');
|
|
// A lock whose PID is gone is not "running" — it is residue from a SIGKILL or a power cut,
|
|
// and saying so is the difference between "stop it" and "clear it".
|
|
if ($pid <= 0 || !is_dir("/proc/$pid")) {
|
|
$out['stale_lock'] = true;
|
|
$out['pid'] = $pid ?: null;
|
|
return $out;
|
|
}
|
|
|
|
$cmd = (string)@file_get_contents("/proc/$pid/cmdline");
|
|
$args = explode("\0", $cmd);
|
|
$out['running'] = true;
|
|
$out['pid'] = $pid;
|
|
$out['mode'] = in_array('--dry-run', $args, true) || in_array('-n', $args, true) ? 'dry-run' : 'live';
|
|
$st = @stat("/proc/$pid");
|
|
if ($st) $out['since'] = (int)$st['mtime'];
|
|
return $out;
|
|
}
|
|
|
|
// Per-host daemon state. Local reads /proc directly; a partner is asked over the same SSH the
|
|
// rest of this file already uses, in one call rather than three.
|
|
function vv_fb_remote_proc(string $ip, string $sshKey): array {
|
|
$cmd = 'for L in fallback fallback_test; do F=/tmp/unraid_locks/$L.lock; '
|
|
. 'if [ -f "$F" ]; then P=$(cut -d: -f1 "$F"); '
|
|
. 'if [ -d "/proc/$P" ]; then M=live; tr "\\0" " " < /proc/$P/cmdline | grep -q -- "--dry-run" && M=dry-run; '
|
|
. 'echo "$L:running:$P:$M"; else echo "$L:stale:$P:"; fi; else echo "$L:none::"; fi; done';
|
|
$out = vv_pt_ssh($ip, $sshKey, $cmd);
|
|
$res = ['fallback' => ['running' => false, 'pid' => null, 'mode' => null, 'stale_lock' => false],
|
|
'fallback_test' => ['running' => false, 'pid' => null, 'mode' => null, 'stale_lock' => false]];
|
|
foreach (explode("\n", trim((string)$out)) as $line) {
|
|
$p = explode(':', trim($line));
|
|
if (count($p) < 4 || !isset($res[$p[0]])) continue;
|
|
if ($p[1] === 'running') {
|
|
$res[$p[0]] = ['running' => true, 'pid' => (int)$p[2], 'mode' => $p[3] ?: 'live', 'stale_lock' => false];
|
|
} elseif ($p[1] === 'stale') {
|
|
$res[$p[0]]['stale_lock'] = true;
|
|
$res[$p[0]]['pid'] = (int)$p[2] ?: null;
|
|
}
|
|
}
|
|
return $res;
|
|
}
|
|
|
|
// A running dry run keeps its own state file — VV_CACHE_ROOT/fallback_state.dryrun.<pid> — and
|
|
// refreshes it every check interval, exactly as the live daemon would. Reading the LIVE file
|
|
// while a preview is running is how a healthy dry run came to render as UNKNOWN on a host that
|
|
// has simply never run fallback for real: the answer existed, in a file next to the one being
|
|
// read. The preview is shown as the preview, never merged into the live state.
|
|
function vv_fb_dryrun_state(int $pid): array {
|
|
$p = (VV_CACHE_ROOT ?: '/tmp/varaverk') . '/fallback_state.dryrun.' . $pid;
|
|
if (!is_file($p)) return ['state' => null, 'age' => null];
|
|
$s = vv_fb_parse_state((string)@file_get_contents($p));
|
|
$s['age'] = time() - (int)@filemtime($p);
|
|
return $s;
|
|
}
|
|
|
|
function vv_fb_remote_dryrun_state(string $ip, string $sshKey, int $pid): array {
|
|
$f = '/tmp/varaverk/fallback_state.dryrun.' . $pid;
|
|
$out = vv_pt_ssh($ip, $sshKey, "[ -f '$f' ] && { echo \"__age=\$(( \$(date +%s) - \$(stat -c %Y '$f') ))\"; cat '$f'; }");
|
|
if (trim((string)$out) === '') return ['state' => null, 'age' => null];
|
|
$age = null;
|
|
if (preg_match('/^__age=(\d+)/m', (string)$out, $m)) $age = (int)$m[1];
|
|
$s = vv_fb_parse_state((string)$out);
|
|
$s['age'] = $age;
|
|
return $s;
|
|
}
|
|
// ── Covers — what a node runs for the other when it's down ───────────────────
|
|
|
|
function vv_fb_covers(string $covering, string $remote, string $coveringRaw, string $remoteRaw): array {
|
|
$ru = strtoupper($remote); // HOST2 — covered host owns its own tier lists
|
|
$tiers = [];
|
|
for ($t = 1; $t <= 4; $t++) {
|
|
$tiers["tier$t"] = vv_fb_bash_array($remoteRaw, "FALLBACK_{$ru}_TIER{$t}");
|
|
}
|
|
// Delays: how long the remote (covered) host must be down before each tier fires.
|
|
// Stored in the *remote* host's conf as REMOTE_TIER*_DELAY.
|
|
$tiers['delays'] = [
|
|
'tier2' => (int)(vv_fb_scalar($remoteRaw, "{$ru}_TIER2_DELAY") ?: 240),
|
|
'tier3' => (int)(vv_fb_scalar($remoteRaw, "{$ru}_TIER3_DELAY") ?: 720),
|
|
'tier4' => (int)(vv_fb_scalar($remoteRaw, "{$ru}_TIER4_DELAY") ?: 1440),
|
|
];
|
|
return $tiers;
|
|
}
|
|
|
|
// ── Known hosts ───────────────────────────────────────────────────────────────
|
|
|
|
function vv_fb_known_hosts(): array {
|
|
return vv_known_hosts();
|
|
}
|
|
|
|
// ── Main data builder ─────────────────────────────────────────────────────────
|
|
|
|
function vv_fb_all(): array {
|
|
$currentHost = vv_detect_host();
|
|
$hosts = vv_fb_known_hosts();
|
|
$tsPeers = vv_pt_ts_peers();
|
|
$masterRaw = vv_read_conf_raw('master.conf');
|
|
$handbackReq = (int)(vv_fb_scalar($masterRaw, 'FALLBACK_HANDBACK_STRIKES') ?: 3);
|
|
$fbEnabled = vv_fb_scalar($masterRaw, 'FALLBACK_ENABLED') === 'true';
|
|
$ptEnabled = vv_fb_scalar($masterRaw, 'PARTNERSHIP_ENABLED') === 'true';
|
|
$rsyncEnabled = vv_fb_scalar($masterRaw, 'FALLBACK_RSYNC_ENABLED') !== 'false';
|
|
$checkInterval = (int)(vv_fb_scalar($masterRaw, 'FALLBACK_CHECK_INTERVAL') ?: 30);
|
|
$suspendAfter = (int)(vv_fb_scalar($masterRaw, 'FALLBACK_PARTNERSHIP_SUSPEND_AFTER') ?: 120);
|
|
|
|
// Read all host conf raws upfront
|
|
$raws = [];
|
|
foreach (array_keys($hosts) as $slot) {
|
|
$raws[$slot] = vv_read_host_conf_raw($slot); // partner conf lives in the RAM cache, not CONF_DIR
|
|
}
|
|
|
|
// SSH key — from local host conf
|
|
$myId = strtoupper($currentHost);
|
|
$myRaw = $raws[$currentHost] ?? '';
|
|
$mySshKey = vv_fb_scalar($myRaw, $myId . '_SSH_KEY');
|
|
|
|
$nodes = [];
|
|
foreach ($hosts as $slot => $hostname) {
|
|
$isMe = ($slot === $currentHost || $currentHost === 'unknown');
|
|
// Exact-key only, until now — see vv_pt_peer_lookup(). This mesh's conf name and tailnet
|
|
// name differ by one character, so HOST2 missed every lookup and the page rendered a
|
|
// partner that was up the whole time as state=UNREACHABLE.
|
|
$ts = vv_pt_peer_lookup($tsPeers, $hostname);
|
|
$ip = $ts['ip'] ?? null;
|
|
|
|
// State
|
|
if ($isMe) {
|
|
$state = vv_fb_local_state();
|
|
} elseif ($ip && $mySshKey) {
|
|
$state = vv_fb_remote_state($ip, $mySshKey);
|
|
} else {
|
|
$state = vv_fb_parse_state('');
|
|
$state['state'] = $ts['online'] === false ? 'OFFLINE' : 'UNREACHABLE';
|
|
}
|
|
|
|
// Running containers
|
|
if ($isMe) {
|
|
$running = vv_fb_local_running();
|
|
} elseif ($ip && $mySshKey && $ts['online']) {
|
|
$running = vv_fb_remote_running($ip, $mySshKey);
|
|
} else {
|
|
$running = [];
|
|
}
|
|
|
|
// Covers: for a 2-node setup, each covers the other
|
|
// For N nodes this would need a different approach — for now, assume 2-node
|
|
$covers = null;
|
|
foreach ($hosts as $otherSlot => $otherHostname) {
|
|
if ($otherSlot === $slot) continue;
|
|
$coveringRaw = $raws[$slot] ?? '';
|
|
$remoteRaw = $raws[$otherSlot] ?? '';
|
|
$covers = [
|
|
'slot' => $otherSlot,
|
|
'id' => strtoupper($otherSlot),
|
|
'hostname' => $otherHostname,
|
|
] + vv_fb_covers($slot, $otherSlot, $coveringRaw, $remoteRaw);
|
|
break; // 2-node only
|
|
}
|
|
|
|
// Daemon liveness per node. Everything in $state was written by this process — without
|
|
// it a NORMAL left behind by a daemon that died days ago is indistinguishable from a
|
|
// NORMAL being refreshed every 30 seconds.
|
|
if ($isMe) {
|
|
$proc = vv_fb_proc('fallback');
|
|
$procTest = vv_fb_proc('fallback_test');
|
|
} elseif ($ip && $mySshKey && $ts['online']) {
|
|
$rp = vv_fb_remote_proc($ip, $mySshKey);
|
|
$proc = $rp['fallback'];
|
|
$procTest = $rp['fallback_test'];
|
|
} else {
|
|
$proc = ['running' => null, 'pid' => null, 'mode' => null, 'stale_lock' => false];
|
|
$procTest = ['running' => null, 'pid' => null, 'mode' => null, 'stale_lock' => false];
|
|
}
|
|
|
|
// When a preview is running, read what IT is deciding — kept beside the live file and
|
|
// refreshed on the same interval. Reported separately so the live state is never
|
|
// overwritten by a preview's opinion.
|
|
$preview = ['state' => null, 'age' => null];
|
|
if (($proc['mode'] ?? '') === 'dry-run' && !empty($proc['pid'])) {
|
|
$preview = $isMe
|
|
? vv_fb_dryrun_state((int)$proc['pid'])
|
|
: vv_fb_remote_dryrun_state($ip, $mySshKey, (int)$proc['pid']);
|
|
}
|
|
|
|
// How fresh the state actually is. The daemon rewrites its file every check interval, so
|
|
// an age far past that interval means it is wedged even while the process still exists.
|
|
$stateAge = null;
|
|
if ($isMe) {
|
|
$sp = STATE_DIR . '/fallback_state.db';
|
|
if (is_file($sp)) $stateAge = time() - (int)@filemtime($sp);
|
|
}
|
|
|
|
// "Reachable" is three separate facts and one boolean hid which had failed.
|
|
$reach = [
|
|
'tailscale' => $isMe ? true : ($ts['online'] === true),
|
|
'ip' => $isMe ? null : $ip,
|
|
'ssh' => $isMe ? true : ($running !== [] || ($proc['running'] !== null)),
|
|
'state_file' => ($state['state'] ?? 'UNKNOWN') !== 'UNKNOWN',
|
|
];
|
|
|
|
$nodes[] = [
|
|
'slot' => $slot,
|
|
'id' => strtoupper($slot),
|
|
'hostname' => $hostname,
|
|
'is_me' => $isMe,
|
|
'ts_online' => $ts['online'],
|
|
'ts_ip' => $ip,
|
|
'state' => $state,
|
|
'state_age' => $stateAge,
|
|
'preview' => $preview,
|
|
'running' => $running,
|
|
'running_count' => count($running),
|
|
'proc' => $proc,
|
|
'proc_test' => $procTest,
|
|
'reach' => $reach,
|
|
'covers' => $covers,
|
|
];
|
|
}
|
|
|
|
return [
|
|
'ts' => time(),
|
|
'fb_enabled' => $fbEnabled,
|
|
'partnership_enabled' => $ptEnabled,
|
|
'fb_rsync_enabled' => $rsyncEnabled,
|
|
'dry_run' => vv_fb_scalar($masterRaw, 'FALLBACK_DRY_RUN') === 'true',
|
|
'handback_req' => $handbackReq,
|
|
'check_interval' => $checkInterval,
|
|
'suspend_after' => $suspendAfter,
|
|
// Tier delays are per-host and live in this host's own conf, not master.conf — the save
|
|
// path has to name the right file per field or the write is refused as an absent key.
|
|
'my_conf' => vv_detect_host() . '.conf',
|
|
'tier_delays' => [
|
|
'tier2' => (int)(vv_fb_scalar($myRaw, strtoupper($currentHost) . '_TIER2_DELAY') ?: 240),
|
|
'tier3' => (int)(vv_fb_scalar($myRaw, strtoupper($currentHost) . '_TIER3_DELAY') ?: 720),
|
|
'tier4' => (int)(vv_fb_scalar($myRaw, strtoupper($currentHost) . '_TIER4_DELAY') ?: 1440),
|
|
],
|
|
'external_ip' => vv_fb_scalar($masterRaw, 'EXTERNAL_IP'),
|
|
'nodes' => $nodes,
|
|
];
|
|
}
|