631 lines
31 KiB
PHP
631 lines
31 KiB
PHP
<?php
|
||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||
// PURPOSE
|
||
// Partnership page data layer. Enumerates the nodes in the mesh, reaches each over
|
||
// Tailscale, and reports identity, system summary, sync history and reachability — the
|
||
// view of "who is in this partnership and are they alive".
|
||
//
|
||
// DESIGN PRINCIPLES
|
||
// Tailscale is the transport; hostnames resolve through it.
|
||
// Peers come from `tailscale status`, never from hardcoded IPs. A node that moves
|
||
// networks stays reachable because nothing here records where it used to be.
|
||
//
|
||
// The API is tried first, SSH is the fallback.
|
||
// vv_pt_remote_system() prefers the partner's unraid-api and drops to a single
|
||
// combined SSH call for version/uptime/load/containers when the API is unavailable.
|
||
// One round trip either way.
|
||
//
|
||
// Every remote read is one command.
|
||
// Partner data is gathered in a single SSH invocation rather than several, because
|
||
// each one pays full connection setup over a WAN link.
|
||
//
|
||
// OPERATIONAL SAFEGUARDS
|
||
// SSH is time-boxed, non-interactive, and escaped.
|
||
// ConnectTimeout, BatchMode=yes so it can never sit waiting for a password, and every
|
||
// interpolated value passed through escapeshellarg(). A partner that is powered off
|
||
// costs the configured timeout, not a hung page.
|
||
//
|
||
// A missing or unreadable key is treated as unreachable.
|
||
// vv_pt_ssh() returns empty immediately when the key path does not exist, rather than
|
||
// invoking ssh and letting it fail slowly.
|
||
//
|
||
// Unreachable partners degrade per node.
|
||
// Each node is collected independently; one dark host leaves its own card empty and
|
||
// affects nothing else.
|
||
//
|
||
// Read-only over SSH. Commands issued are state reads and inventory — this file never
|
||
// deploys, starts, or stops anything on a partner.
|
||
//
|
||
// EXPORTS
|
||
// Config vv_pt_config(), vv_pt_nodes(), vv_pt_ts_peers()
|
||
// Transport vv_pt_ssh(), vv_pt_ping()
|
||
// System vv_pt_local_system(), vv_pt_remote_system()
|
||
// Sync vv_pt_sync(), vv_pt_sync_summary(), vv_pt_read_db()
|
||
// Assembly vv_partnership_all()
|
||
//
|
||
// CONFIGURATION
|
||
// HOST*_SSH_KEY per-host key used for every partner call
|
||
// HOST*_UNRAID_API_KEY preferred path before SSH fallback
|
||
// PARTNERSHIP_ENABLED whether the partnership layer is active
|
||
// STATE_DIR / DATA_DIR sync history and offline counters
|
||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||
// Partnership page data helpers
|
||
|
||
require_once __DIR__ . '/config.php';
|
||
require_once __DIR__ . '/arrs.php'; // vv_arr_known_hosts(), vv_arr_scalar()
|
||
require_once __DIR__ . '/common.php'; // vv_system_info(), vv_docker_containers(), vv_remote_hosts_stats(), vv_api_node_metrics()
|
||
|
||
// ── Config ────────────────────────────────────────────────────────────────────
|
||
|
||
function vv_pt_config(): array {
|
||
$v = vv_conf_vars();
|
||
// The file is named ..._days.db and does not hold days. partnership_manager.sh --check runs
|
||
// from the 30-minute orchestrator and increments once per run, so the unit is INTERVALS, and
|
||
// the auto-offboard threshold it is compared against is computed as days × 48. Reading the
|
||
// raw number and printing it as days overstated the outage by 48× — a partner unreachable
|
||
// for half an hour rendered as "unreachable 1 day · auto-offboard in 29 days".
|
||
$offlineDays = null;
|
||
$offlineIntervals = null;
|
||
$odFile = STATE_DIR . '/partnership_offline_days.db';
|
||
if (file_exists($odFile)) {
|
||
$raw = trim(@file_get_contents($odFile) ?: '');
|
||
if (is_numeric($raw)) {
|
||
$offlineIntervals = (int)$raw;
|
||
$offlineDays = $offlineIntervals / 48;
|
||
}
|
||
}
|
||
return [
|
||
'enabled' => ($v['PARTNERSHIP_ENABLED'] ?? 'false') === 'true',
|
||
'owner_host' => $v['PARTNERSHIP_OWNER_HOST'] ?? '',
|
||
'sync_min' => (int)($v['PARTNERSHIP_SYNC_INTERVAL'] ?? 15),
|
||
'grace_hours' => (int)($v['PARTNERSHIP_GRACE_HOURS'] ?? 6),
|
||
'offline_threshold' => (int)($v['PARTNERSHIP_OFFLINE_THRESHOLD'] ?? 30),
|
||
'offline_days' => $offlineDays,
|
||
'offline_intervals' => $offlineIntervals,
|
||
'remove_tailscale' => ($v['PARTNERSHIP_REMOVE_TAILSCALE'] ?? 'false') === 'true',
|
||
'folderview3' => ($v['PARTNERSHIP_FOLDERVIEW3'] ?? 'false') === 'true',
|
||
'tailscale_configured' => !empty($v['TAILSCALE_API_KEY']) && !empty($v['TAILSCALE_TAILNET']),
|
||
'transfer_confirm' => $v['PARTNERSHIP_TRANSFER_CONFIRM'] ?? 'i-understand-this-transfers-ownership',
|
||
];
|
||
}
|
||
|
||
// ── Mirror sync health — the partnership's actual job (rsync orchestrators) ──────
|
||
|
||
function vv_pt_sync_summary(string $logFile): string {
|
||
if (!file_exists($logFile)) return '';
|
||
$tail = array_filter(array_slice(file($logFile, FILE_IGNORE_NEW_LINES), -30), 'strlen');
|
||
foreach (array_reverse(array_values($tail)) as $raw) {
|
||
// Strip emoji / Unicode decoration for regex matching
|
||
$line = trim(preg_replace('/[\x{1F000}-\x{1FFFF}\x{2600}-\x{27BF}\x{FE0F}]/u', '', $raw));
|
||
$line = preg_replace('/\s+/', ' ', $line);
|
||
// "Critical sync complete — HOST1 — 1m39s — 2 share(s)"
|
||
if (preg_match('/Critical sync complete\s*—\s*\S+\s*—\s*([\w]+)\s*—\s*(.+)/i', $line, $m))
|
||
return trim($m[2]) . ' in ' . $m[1];
|
||
// "Status: all complete — 0 share(s) synced, 11 job(s) run"
|
||
if (preg_match('/Status:\s*all complete\s*—\s*(.+)/i', $line, $m))
|
||
return trim($m[1]);
|
||
// "Status: X failure(s)" or "Failures: N"
|
||
if (preg_match('/Failures:\s*(\d+)/i', $line, $m) && (int)$m[1] > 0)
|
||
return (int)$m[1] . ' failure(s)';
|
||
}
|
||
return '';
|
||
}
|
||
|
||
function vv_pt_sync(): array {
|
||
$jobs = [
|
||
'critical' => 'Orchestrators/critical_sync_maintenance',
|
||
'daily' => 'Orchestrators/daily_sync_maintenance',
|
||
'weekly' => 'Orchestrators/weekly_sync_maintenance',
|
||
];
|
||
$out = ['jobs' => []];
|
||
foreach ($jobs as $key => $base) {
|
||
$statFile = LOG_DIR . '/' . $base . '.json';
|
||
$logFile = LOG_DIR . '/' . $base . '.log';
|
||
$s = file_exists($statFile) ? json_decode(@file_get_contents($statFile), true) : null;
|
||
$out['jobs'][$key] = is_array($s) ? [
|
||
'status' => $s['status'] ?? 'unknown',
|
||
'start' => isset($s['start']) ? (int)$s['start'] : null,
|
||
'end' => isset($s['end']) ? (int)$s['end'] : null,
|
||
'summary' => vv_pt_sync_summary($logFile),
|
||
] : null;
|
||
}
|
||
$v = vv_conf_vars();
|
||
$out['interval_min'] = (int)($v['PARTNERSHIP_SYNC_INTERVAL'] ?? 30);
|
||
// Rsync gate flags (master.conf) — global Tier 1 + per-tier Tier 2.
|
||
$out['gates'] = [
|
||
'global' => ['var' => 'RSYNC_ENABLED', 'on' => ($v['RSYNC_ENABLED'] ?? 'true') === 'true'],
|
||
'critical' => ['var' => 'CRITICAL_RSYNC_ENABLED', 'on' => ($v['CRITICAL_RSYNC_ENABLED'] ?? 'true') === 'true'],
|
||
'daily' => ['var' => 'DAILY_RSYNC_ENABLED', 'on' => ($v['DAILY_RSYNC_ENABLED'] ?? 'true') === 'true'],
|
||
'weekly' => ['var' => 'WEEKLY_RSYNC_ENABLED', 'on' => ($v['WEEKLY_RSYNC_ENABLED'] ?? 'true') === 'true'],
|
||
];
|
||
// Back-compat keys still used by the warning line.
|
||
$out['rsync_enabled'] = $out['gates']['global']['on'];
|
||
$out['critical_enabled'] = $out['gates']['critical']['on'];
|
||
return $out;
|
||
}
|
||
|
||
// ── State file parser ─────────────────────────────────────────────────────────
|
||
|
||
function vv_pt_read_db(string $path): array {
|
||
if (!file_exists($path)) return [];
|
||
$out = [];
|
||
foreach (file($path, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) as $line) {
|
||
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
||
if ($k) $out[trim($k)] = trim($v, '"\'');
|
||
}
|
||
return $out;
|
||
}
|
||
|
||
// ── Tailscale peers ───────────────────────────────────────────────────────────
|
||
|
||
function vv_pt_ts_peers(): array {
|
||
$raw = shell_exec('tailscale status --json 2>/dev/null') ?: '{}';
|
||
$data = json_decode($raw, true) ?: [];
|
||
$peers = [];
|
||
|
||
// Self
|
||
$self = $data['Self'] ?? [];
|
||
$selfLabel = strtolower(explode('.', $self['DNSName'] ?? '')[0]);
|
||
if ($selfLabel) {
|
||
$peers[$selfLabel] = [
|
||
'online' => true,
|
||
'active' => true,
|
||
'ip' => $self['TailscaleIPs'][0] ?? null,
|
||
];
|
||
}
|
||
|
||
// Peers
|
||
foreach ($data['Peer'] ?? [] as $peer) {
|
||
$label = strtolower(explode('.', $peer['DNSName'] ?? '')[0]);
|
||
if (!$label) continue;
|
||
$peers[$label] = [
|
||
'online' => (bool)($peer['Online'] ?? false),
|
||
'active' => (bool)($peer['Active'] ?? false),
|
||
'ip' => $peer['TailscaleIPs'][0] ?? null,
|
||
];
|
||
}
|
||
return $peers;
|
||
}
|
||
|
||
// ── SSH helper — run a single command on a remote host ────────────────────────
|
||
|
||
function vv_pt_ssh(string $ip, string $sshKey, string $cmd, int $timeout = 4): string {
|
||
if (!$ip || !$sshKey || !file_exists($sshKey)) return '';
|
||
$full = sprintf(
|
||
'ssh -i %s -o ConnectTimeout=%d -o StrictHostKeyChecking=no -o BatchMode=yes root@%s %s 2>/dev/null',
|
||
escapeshellarg($sshKey), $timeout, escapeshellarg($ip), escapeshellarg($cmd)
|
||
);
|
||
return shell_exec($full) ?: '';
|
||
}
|
||
|
||
// ── System info ───────────────────────────────────────────────────────────────
|
||
|
||
// vv_system_info() (common.php) provides version, load_avg, array_state.
|
||
// /proc/uptime is the reliable uptime source (API uptime is an ISO date string, not seconds).
|
||
// vv_docker_containers() (common.php) provides the running container list.
|
||
function vv_pt_local_system(): array {
|
||
$info = vv_system_info();
|
||
$uptimeSec = file_exists('/proc/uptime')
|
||
? (int)explode(' ', file_get_contents('/proc/uptime'))[0] : 0;
|
||
return [
|
||
'unraid_version' => $info['version'] ?? '',
|
||
'uptime_sec' => $uptimeSec,
|
||
'load_avg' => isset($info['load_avg']) ? $info['load_avg'][0] : null,
|
||
'containers' => count(vv_docker_containers()),
|
||
];
|
||
}
|
||
|
||
// SSH fallback for remote nodes — version/uptime/load/containers in one call.
|
||
// API stats from vv_remote_hosts_stats() take priority when available; SSH fills gaps.
|
||
function vv_pt_remote_system(string $ip, string $sshKey): array {
|
||
$out = vv_pt_ssh($ip, $sshKey,
|
||
'printf "%s\nUPTIME:%s\nLOAD:%s\nCONTAINERS:%s\n" ' .
|
||
'"$(cat /etc/unraid-version 2>/dev/null)" ' .
|
||
'"$(cat /proc/uptime 2>/dev/null)" ' .
|
||
'"$(awk \'{print $1}\' /proc/loadavg 2>/dev/null)" ' .
|
||
'"$(docker ps -q 2>/dev/null | wc -l)"');
|
||
$ver = '';
|
||
preg_match('/VERSION="([^"]+)"/', $out, $m); if ($m) $ver = $m[1];
|
||
$uptime = 0;
|
||
if (preg_match('/UPTIME:([\d.]+)/', $out, $m)) $uptime = (int)$m[1];
|
||
$load = null;
|
||
if (preg_match('/LOAD:([\d.]+)/', $out, $m)) $load = round((float)$m[1], 2);
|
||
$containers = null;
|
||
if (preg_match('/CONTAINERS:(\d+)/', $out, $m)) $containers = (int)$m[1];
|
||
return [
|
||
'unraid_version' => $ver,
|
||
'uptime_sec' => $uptime,
|
||
'load_avg' => $load,
|
||
'containers' => $containers,
|
||
];
|
||
}
|
||
|
||
// ── Transfer volume ───────────────────────────────────────────────────────────
|
||
//
|
||
// Reads data/db/bandwidth_history.db, one line per completed rsync:
|
||
// YYYY-MM-DD|HH:MM|profile|duration|status|bytes
|
||
//
|
||
// Rows before VV_BW_EPOCH are excluded outright. Until 2026-08-14 rsync.sh read the byte count
|
||
// from the wrong field of rsync --stats and every row was written as 0 — 1,423 of them. Summing
|
||
// those produces a confident 0.00 GB for any window that reaches back far enough, which is worse
|
||
// than an empty card: it is a measurement that looks taken. The cut is on the fix, not on a
|
||
// guess about which rows look plausible.
|
||
//
|
||
// A window whose rows are all zero reports null, not 0, so the UI can say "no transfers
|
||
// recorded" rather than assert nothing moved. Those are different claims and only one of them
|
||
// is supported by an all-zero column.
|
||
define('VV_BW_EPOCH', '2026-08-14');
|
||
|
||
function vv_pt_transfer_stats(): array {
|
||
$file = DATA_DIR . '/db/bandwidth_history.db';
|
||
$out = ['epoch' => VV_BW_EPOCH, 'windows' => [], 'live' => null, 'usable_rows' => 0];
|
||
|
||
$windows = ['24h' => 86400, '7d' => 604800, '30d' => 2592000];
|
||
foreach ($windows as $k => $_) $out['windows'][$k] = ['bytes' => null, 'runs' => 0, 'failed' => 0];
|
||
|
||
if (is_file($file)) {
|
||
$now = time();
|
||
$epoch = strtotime(VV_BW_EPOCH . ' 00:00:00');
|
||
$fh = fopen($file, 'r');
|
||
if ($fh) {
|
||
while (($line = fgets($fh)) !== false) {
|
||
$p = explode('|', trim($line));
|
||
if (count($p) < 6) continue;
|
||
$ts = strtotime($p[0] . ' ' . $p[1]);
|
||
if (!$ts || $ts < $epoch) continue;
|
||
$out['usable_rows']++;
|
||
$bytes = (int)$p[5];
|
||
$failed = ($p[4] ?? '') !== 'success';
|
||
foreach ($windows as $k => $span) {
|
||
if ($ts < $now - $span) continue;
|
||
$w = &$out['windows'][$k];
|
||
$w['runs']++;
|
||
if ($failed) $w['failed']++;
|
||
if ($bytes > 0) $w['bytes'] = (int)$w['bytes'] + $bytes;
|
||
unset($w);
|
||
}
|
||
}
|
||
fclose($fh);
|
||
}
|
||
}
|
||
|
||
// Live: an rsync this host is running right now. The lock name carries the share, which is
|
||
// the only part of "what is moving" worth showing without parsing progress output.
|
||
$running = trim((string)shell_exec("pgrep -c -f '^rsync -av' 2>/dev/null")) ;
|
||
if ((int)$running > 0) {
|
||
$share = '';
|
||
foreach (glob('/tmp/unraid_locks/rsync_*.lock') ?: [] as $l) {
|
||
$pid = (int)preg_replace('/\D/', '', (string)@file_get_contents($l));
|
||
if ($pid && is_dir("/proc/$pid")) {
|
||
$share = preg_replace('/^rsync_|\.lock$/', '', basename($l));
|
||
break;
|
||
}
|
||
}
|
||
$out['live'] = ['running' => (int)$running, 'share' => $share];
|
||
}
|
||
return $out;
|
||
}
|
||
|
||
// ── Media seed progress ───────────────────────────────────────────────────────
|
||
//
|
||
// Phase 3 dispatches Rsync/media_seed.sh detached, because a first seed of a full
|
||
// media library is a multi-week transfer and used to hold the onboard — and therefore the
|
||
// phase-2 flag, and therefore this whole page — open for the duration.
|
||
//
|
||
// Detaching it means nothing on screen would mention it at all unless something reads its
|
||
// job record, which is what this does. The seed is a push from this host to the partner, so
|
||
// it is rendered on the partner's card even though every byte of evidence for it is local.
|
||
//
|
||
// Returns [] when there has never been a seed. A stale "running" record whose pid is gone is
|
||
// reported as stopped rather than running: a record is not a process.
|
||
function vv_pt_media_seed(array $setupDb = [], string $nodeIdUpper = ''): array {
|
||
$stat = '/var/log/varaverk/Rsync/media_seed.json';
|
||
$log = '/var/log/varaverk/Rsync/media_seed.log';
|
||
|
||
// MEDIA_SEED_ENABLED is reported alongside the record rather than instead of it, because
|
||
// the two answer different questions: the toggle says whether a seed may start, the record
|
||
// says what the last one did. media_seed.sh exits 0 when the toggle is off — correct, it is
|
||
// a decision and not a failure — so a record reading "ok" beside a disabled toggle would
|
||
// otherwise render as "Seed complete" over a partner that was never seeded.
|
||
//
|
||
// Unset reads as enabled, matching media_seed.sh: the toggle postdates the script.
|
||
$raw = vv_read_conf_raw('master.conf');
|
||
$toggle = vv_arr_scalar($raw, 'MEDIA_SEED_ENABLED');
|
||
$enabled = ($toggle === '' || strtolower($toggle) === 'true');
|
||
|
||
// Phase 3 is complete only when every share landed, which media_seed.sh decides and records
|
||
// at the end of a run that may have taken weeks. The job record cannot answer this: it goes
|
||
// "ok" the moment the process exits, including the exits that seeded nothing.
|
||
$phase3Done = $nodeIdUpper !== ''
|
||
&& ($setupDb[$nodeIdUpper . '_PHASE3_DONE'] ?? '') === 'true';
|
||
|
||
if (!is_file($stat)) return ['enabled' => $enabled, 'phase3_done' => $phase3Done];
|
||
|
||
$j = json_decode((string)file_get_contents($stat), true);
|
||
if (!is_array($j)) return ['enabled' => $enabled, 'phase3_done' => $phase3Done];
|
||
|
||
$status = (string)($j['status'] ?? '');
|
||
$pid = (int)($j['pid'] ?? 0);
|
||
if ($status === 'running' && (!$pid || !is_dir("/proc/$pid"))) {
|
||
$status = 'stopped';
|
||
}
|
||
|
||
$share = '';
|
||
$line = '';
|
||
if (is_file($log)) {
|
||
// rsync --info=progress2 rewrites one line with \r, so the tail is read as bytes and
|
||
// split on both terminators — splitting on \n alone yields a single enormous "line"
|
||
// holding every progress repaint of the current file.
|
||
$fh = fopen($log, 'r');
|
||
if ($fh) {
|
||
fseek($fh, 0, SEEK_END);
|
||
$size = ftell($fh);
|
||
fseek($fh, max(0, $size - 65536));
|
||
$tail = (string)fread($fh, 65536);
|
||
fclose($fh);
|
||
$parts = preg_split('/[\r\n]+/', $tail) ?: [];
|
||
for ($i = count($parts) - 1; $i >= 0; $i--) {
|
||
$p = trim($parts[$i]);
|
||
if ($p === '') continue;
|
||
if ($line === '') $line = $p;
|
||
if (preg_match('/Seeding:\s*(\S+)/', $p, $m)) { $share = $m[1]; break; }
|
||
}
|
||
}
|
||
}
|
||
|
||
return array_filter([
|
||
'enabled' => $enabled,
|
||
'phase3_done' => $phase3Done,
|
||
'status' => $status,
|
||
'start' => isset($j['start']) ? (int)$j['start'] : null,
|
||
'end' => isset($j['end']) ? (int)$j['end'] : null,
|
||
'share' => $share,
|
||
'line' => mb_substr($line, 0, 160),
|
||
], fn($v) => $v !== null && $v !== '');
|
||
}
|
||
|
||
// ── Per-node data ─────────────────────────────────────────────────────────────
|
||
|
||
function vv_pt_nodes(): array {
|
||
$currentHost = vv_detect_host();
|
||
$hosts = vv_arr_known_hosts(); // ['host1' => 'hostname', ...]
|
||
$vars = vv_conf_vars();
|
||
$tsPeers = vv_pt_ts_peers();
|
||
// Read once: the partnership state this host records for itself, used by every remote's
|
||
// mesh check below.
|
||
$localPtState = vv_pt_read_db(STATE_DIR . '/partnership_' . vv_get_hostname() . '.db')['state'] ?? '';
|
||
$ownerSlot = strtolower($vars['PARTNERSHIP_OWNER_HOST'] ?? '');
|
||
$setupDb = vv_setup_state_read();
|
||
|
||
// Remote host stats (API + 30s /tmp cache) — includes version, uptime, cpu/ram/array/temp/vms
|
||
$remoteStats = vv_remote_hosts_stats();
|
||
|
||
// SSH key for this host
|
||
$myId = strtoupper($currentHost);
|
||
$myRaw = vv_read_conf_raw($currentHost . '.conf');
|
||
$mySshKey = vv_arr_scalar($myRaw, $myId . '_SSH_KEY');
|
||
|
||
$nodes = [];
|
||
foreach ($hosts as $slot => $hostname) {
|
||
$isMe = ($slot === $currentHost || $currentHost === 'unknown');
|
||
$isOwner = (strtolower($ownerSlot) === $slot);
|
||
|
||
// Tailscale
|
||
//
|
||
// Exact key first, then a single unambiguous prefix match in either direction. The
|
||
// tailnet name and the OS hostname are not the same string and nothing keeps them in
|
||
// step: this mesh has master.conf saying "unRAID-Jayred36" while the tailnet device is
|
||
// "unraid-jayred365". An exact-key lookup found nothing, so every remote card rendered
|
||
// with no IP, no online state and no container count — a partner that was answering SSH
|
||
// the whole time displayed as though it were not there.
|
||
//
|
||
// Same rule as vv_resolve_tailscale_ip(), and the same refusal: one candidate or none.
|
||
// server1 must never resolve to server10 because it happens to share a prefix.
|
||
$tsLabel = strtolower($hostname);
|
||
$ts = $tsPeers[$tsLabel] ?? null;
|
||
if ($ts === null) {
|
||
$cand = [];
|
||
foreach ($tsPeers as $peerName => $peer) {
|
||
if (str_starts_with($peerName, $tsLabel) || str_starts_with($tsLabel, $peerName)) {
|
||
$cand[] = $peer;
|
||
}
|
||
}
|
||
if (count($cand) === 1) $ts = $cand[0];
|
||
}
|
||
if ($ts === null) $ts = ['online' => null, 'active' => false, 'ip' => null];
|
||
|
||
// Fallback state
|
||
$fbState = 'UNKNOWN';
|
||
if ($isMe) {
|
||
$fb = vv_pt_read_db(STATE_DIR . '/fallback_state.db');
|
||
$fbState = $fb['state'] ?? 'UNKNOWN';
|
||
} elseif ($ts['online'] && $ts['ip'] && $mySshKey) {
|
||
$out = vv_pt_ssh($ts['ip'], $mySshKey, vv_remote_state_cmd('fallback_state.db'));
|
||
if ($out) {
|
||
$fb = [];
|
||
foreach (explode("\n", $out) as $line) {
|
||
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
||
if ($k) $fb[trim($k)] = trim($v, '"\'');
|
||
}
|
||
$fbState = $fb['state'] ?? 'UNKNOWN';
|
||
}
|
||
}
|
||
|
||
// Partnership DB — local only (each server writes its own)
|
||
$dbPath = STATE_DIR . "/partnership_{$hostname}.db";
|
||
$ptDb = vv_pt_read_db($dbPath);
|
||
|
||
// System info
|
||
$system = $isMe
|
||
? vv_pt_local_system()
|
||
: ($ts['online'] && $ts['ip'] && $mySshKey ? vv_pt_remote_system($ts['ip'], $mySshKey) : []);
|
||
|
||
// Onboard phase from setup.db — 0=not started, 1=SSH+conf done, 2=fully onboarded.
|
||
//
|
||
// Computed for self as well as for partners. The flags are keyed to the host they
|
||
// describe, so on the mirror HOST2_PHASE*_DONE is the mirror's own progress — and the
|
||
// mirror's action panel needs it to know whether its SSH key step is still outstanding.
|
||
// This was forced to null for self, which left a mirror with no way to see its own
|
||
// phase and no way to render anything but a bare button.
|
||
$nodeIdUpper = strtoupper($slot);
|
||
$onboardPhase = ($setupDb[$nodeIdUpper . '_PHASE2_DONE'] ?? '') === 'true' ? 2
|
||
: (($setupDb[$nodeIdUpper . '_PHASE1_DONE'] ?? '') === 'true' ? 1 : 0);
|
||
|
||
// An ACTIVE partnership means phase 2, whatever the flags say. HOST<n>_PHASE*_DONE always
|
||
// describe the MIRROR — they are the owner's record of how far it got provisioning the
|
||
// other side — so the owner's own slot has no such flag and never will. On the mirror
|
||
// that left the owner's card computing phase 0 and rendering "Not provisioned · SSH keys
|
||
// not installed", complete with an ▶ Onboard button, on the very host that had just
|
||
// finished onboarding it. The state file is the authority on whether a partnership
|
||
// exists; the flags only say who did what to whom.
|
||
if ($onboardPhase < 2 && ($ptDb['state'] ?? '') === 'ACTIVE') {
|
||
$onboardPhase = 2;
|
||
}
|
||
// key_ready: local key generated but not yet installed on HOST2 (SSH pending manual step)
|
||
$keyReady = !$isMe && ($setupDb[$nodeIdUpper . '_KEY_READY'] ?? '') === 'true';
|
||
// For self: local setup complete flag (set by partnership_manager --onboard --local-only)
|
||
$localDone = $isMe && ($setupDb[$nodeIdUpper . '_LOCAL_DONE'] ?? '') === 'true';
|
||
|
||
// Unraid API key status — checks Unraid's key store directly so deletions are reflected.
|
||
$apiKeySet = false;
|
||
$apiKeyPreview = '';
|
||
if ($isMe) {
|
||
$hn = trim((string)shell_exec("hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//'")) ?: 'Varaverk';
|
||
$keyName = 'Varaverk ' . $hn;
|
||
$apiOut = shell_exec('/usr/local/sbin/unraid-api apikey --name ' . escapeshellarg($keyName) . ' --json </dev/null 2>/dev/null');
|
||
$apiData = json_decode(trim($apiOut ?? ''), true);
|
||
if (is_array($apiData) && !empty($apiData['key'])) {
|
||
$apiKeySet = true;
|
||
$apiKeyPreview = substr($apiData['key'], 0, 8) . '...' . substr($apiData['key'], -4);
|
||
}
|
||
} else {
|
||
// The partner's key is in the partner's own conf, which this host holds in the RAM
|
||
// cache that conf_sync fills — never on disk here, because sparse checkout means a
|
||
// host only ever checks out its own host*.conf.
|
||
//
|
||
// Two separate facts, deliberately not merged: a key being CONFIGURED is read from
|
||
// that conf, and the key WORKING is proven by the partner's API having answered this
|
||
// page's own stats call. A configured key that no longer authenticates would
|
||
// otherwise render exactly like a healthy one.
|
||
$pConf = VV_CONF_RAM_CACHE_DIR . '/' . strtolower($nodeIdUpper) . '.conf';
|
||
if (is_readable($pConf)) {
|
||
$pKey = vv_arr_scalar((string)@file_get_contents($pConf), $nodeIdUpper . '_UNRAID_API_KEY');
|
||
if ($pKey !== '') {
|
||
$apiKeySet = true;
|
||
$apiKeyPreview = substr($pKey, 0, 8) . '...' . substr($pKey, -4);
|
||
}
|
||
}
|
||
}
|
||
// Did the partner's API actually answer? Empty for self, where the local API has its own
|
||
// banner on the Monitor tab.
|
||
$apiLive = !$isMe && !empty(($remoteStats[$nodeIdUpper] ?? [])['available']);
|
||
|
||
// Live metrics: local uses vv_api_data() (cached); remote uses vv_remote_hosts_stats() (30s cache)
|
||
if ($isMe) {
|
||
$metrics = array_merge(
|
||
vv_api_node_metrics(vv_api_data()),
|
||
array_filter([
|
||
'load_avg' => $system['load_avg'] ?? null,
|
||
'containers' => $system['containers'] ?? null,
|
||
], fn($v) => $v !== null)
|
||
);
|
||
} else {
|
||
$rStat = $remoteStats[$nodeIdUpper] ?? [];
|
||
// Merge API metrics from remote stats with SSH extras (load, containers)
|
||
$metrics = array_filter([
|
||
'cpu_pct' => $rStat['cpu_pct'] ?? null,
|
||
'ram_used_gb' => $rStat['ram_used_gb'] ?? null,
|
||
'ram_total_gb' => $rStat['ram_total_gb'] ?? null,
|
||
'array_used_tb' => $rStat['array_used_tb'] ?? null,
|
||
'array_total_tb' => $rStat['array_total_tb'] ?? null,
|
||
'max_disk_temp' => $rStat['max_disk_temp'] ?? null,
|
||
'vm_count' => $rStat['vm_count'] ?? null,
|
||
'load_avg' => $system['load_avg'] ?? null,
|
||
'containers' => $system['containers'] ?? null,
|
||
], fn($v) => $v !== null);
|
||
// Fill version/uptime from API stats if SSH didn't provide them
|
||
if (empty($system['unraid_version']) && !empty($rStat['version'])) {
|
||
$system['unraid_version'] = $rStat['version'];
|
||
}
|
||
if (empty($system['uptime_sec']) && !empty($rStat['uptime_sec'])) {
|
||
$system['uptime_sec'] = $rStat['uptime_sec'];
|
||
}
|
||
}
|
||
|
||
$nodes[] = [
|
||
'slot' => $slot,
|
||
'id' => $nodeIdUpper,
|
||
'hostname' => $hostname,
|
||
'is_me' => $isMe,
|
||
'is_owner' => $isOwner,
|
||
'ts_online' => $ts['online'],
|
||
'ts_active' => $ts['active'],
|
||
'ts_ip' => $ts['ip'],
|
||
'fallback' => $fbState,
|
||
'partnership' => $ptDb,
|
||
'system' => $system,
|
||
'onboard_phase' => $onboardPhase,
|
||
'key_ready' => $keyReady,
|
||
'local_done' => $localDone,
|
||
'api_key_set' => $apiKeySet,
|
||
'api_key_preview' => $apiKeyPreview,
|
||
'api_live' => $apiLive,
|
||
'metrics' => $metrics,
|
||
'media_seed' => $isMe ? [] : vv_pt_media_seed($setupDb, $nodeIdUpper),
|
||
// Every condition that has to hold for this partner to actually be usable, each
|
||
// reported separately. One rolled-up boolean would have said "not ok" for the last
|
||
// hour without saying that the only failing part was a name lookup.
|
||
'mesh' => $isMe ? null : [
|
||
'tailscale' => $ts['online'] === true,
|
||
'ssh' => !empty($system['unraid_version']) || !empty($system['uptime_sec']),
|
||
// This host's own state file, not one named after the remote. A partnership is a
|
||
// single mutual fact, and each host writes it under its OWN hostname — so
|
||
// partnership_<remote>.db does not exist here, and reading it made a healthy
|
||
// partnership report as a failing leg on the owner's own page.
|
||
'partnership' => $localPtState === 'ACTIVE',
|
||
'onboarded' => $onboardPhase >= 2,
|
||
],
|
||
];
|
||
}
|
||
return $nodes;
|
||
}
|
||
|
||
// ── Connectivity test — SSH echo with round-trip timing ─────────────────────────
|
||
function vv_pt_ping(string $slot): array {
|
||
$slot = strtolower($slot);
|
||
$vars = vv_conf_vars();
|
||
$hostname = $vars[strtoupper($slot)] ?? '';
|
||
if (!$hostname) return ['ok' => false, 'error' => 'Unknown host slot'];
|
||
|
||
$currentHost = vv_detect_host();
|
||
$myRaw = vv_read_conf_raw($currentHost . '.conf');
|
||
$sshKey = vv_arr_scalar($myRaw, strtoupper($currentHost) . '_SSH_KEY');
|
||
if (!$sshKey || !file_exists($sshKey)) {
|
||
return ['ok' => false, 'error' => 'No SSH key configured on this host'];
|
||
}
|
||
|
||
$ip = vv_resolve_tailscale_ip($hostname);
|
||
if (!$ip) return ['ok' => false, 'error' => "Cannot resolve Tailscale IP for $hostname"];
|
||
|
||
$t0 = microtime(true);
|
||
$out = vv_pt_ssh($ip, $sshKey, 'echo ok', 8);
|
||
$ms = (int)round((microtime(true) - $t0) * 1000);
|
||
|
||
if (trim($out) === 'ok') {
|
||
return ['ok' => true, 'latency_ms' => $ms, 'host' => $hostname, 'ip' => $ip];
|
||
}
|
||
return ['ok' => false, 'error' => "SSH to $hostname ($ip) failed or timed out", 'host' => $hostname];
|
||
}
|
||
|
||
// ── Entry point ───────────────────────────────────────────────────────────────
|
||
|
||
function vv_partnership_all(): array {
|
||
return [
|
||
'config' => vv_pt_config(),
|
||
'nodes' => vv_pt_nodes(),
|
||
'sync' => vv_pt_sync(),
|
||
'xfer' => vv_pt_transfer_stats(),
|
||
'ts' => time(),
|
||
];
|
||
}
|