312 lines
12 KiB
PHP
312 lines
12 KiB
PHP
<?php
|
|
require_once __DIR__ . '/config.php';
|
|
|
|
// Monitor helpers — docker, GPU, resources, transcode sessions, fallback state.
|
|
|
|
function vv_docker_containers(): array {
|
|
$out = shell_exec('docker ps --format \'{"name":"{{.Names}}","status":"{{.Status}}","image":"{{.Image}}"}\' 2>/dev/null');
|
|
$containers = [];
|
|
foreach (explode("\n", trim($out ?? '')) as $line) {
|
|
if (!$line) continue;
|
|
$c = json_decode($line, true);
|
|
if ($c) $containers[] = $c;
|
|
}
|
|
return $containers;
|
|
}
|
|
|
|
function vv_docker_stopped(): array {
|
|
$out = shell_exec('docker ps -a --filter "status=exited" --filter "status=created" --format \'{"name":"{{.Names}}","status":"{{.Status}}"}\' 2>/dev/null');
|
|
$containers = [];
|
|
foreach (explode("\n", trim($out ?? '')) as $line) {
|
|
if (!$line) continue;
|
|
$c = json_decode($line, true);
|
|
if ($c) $containers[] = $c;
|
|
}
|
|
return $containers;
|
|
}
|
|
|
|
function vv_gpu_stats(): array {
|
|
$out = shell_exec('nvidia-smi --query-gpu=name,memory.used,memory.total,utilization.gpu,temperature.gpu,power.draw,utilization.encoder,utilization.decoder --format=csv,noheader,nounits 2>/dev/null');
|
|
if (!$out) return ['available' => false];
|
|
|
|
$parts = array_map('trim', explode(',', $out));
|
|
$power = is_numeric($parts[5] ?? '') ? round((float)$parts[5], 1) : null;
|
|
return [
|
|
'available' => true,
|
|
'name' => $parts[0] ?? '',
|
|
'memory_used' => (int)($parts[1] ?? 0),
|
|
'memory_total' => (int)($parts[2] ?? 0),
|
|
'utilization' => (int)($parts[3] ?? 0),
|
|
'temperature' => (int)($parts[4] ?? 0),
|
|
'power_w' => $power,
|
|
'enc_pct' => (int)($parts[6] ?? 0),
|
|
'dec_pct' => (int)($parts[7] ?? 0),
|
|
];
|
|
}
|
|
|
|
function vv_gpu_processes(): array {
|
|
$out = shell_exec('nvidia-smi --query-compute-apps=pid,used_gpu_memory,name --format=csv,noheader,nounits 2>/dev/null');
|
|
$procs = [];
|
|
foreach (explode("\n", trim($out ?? '')) as $line) {
|
|
if (!$line) continue;
|
|
$parts = array_map('trim', explode(',', $line));
|
|
$procs[] = [
|
|
'pid' => $parts[0] ?? '',
|
|
'memory_mb' => $parts[1] ?? '',
|
|
'name' => $parts[2] ?? '',
|
|
];
|
|
}
|
|
return $procs;
|
|
}
|
|
|
|
function vv_system_resources(): array {
|
|
$mem = [];
|
|
foreach (file('/proc/meminfo') ?: [] as $line) {
|
|
if (preg_match('/^(MemTotal|MemAvailable):\s+(\d+)/', $line, $m))
|
|
$mem[$m[1]] = (int)$m[2];
|
|
}
|
|
return [
|
|
'ram_total_mb' => (int)(($mem['MemTotal'] ?? 0) / 1024),
|
|
'ram_free_mb' => (int)(($mem['MemAvailable'] ?? 0) / 1024),
|
|
'cache' => vv_df('/mnt/cache'),
|
|
];
|
|
}
|
|
|
|
function vv_cpu_per_core(): array {
|
|
// Parse /proc/stat — [user, nice, system, idle, iowait, irq, softirq]
|
|
$raw = [];
|
|
foreach (file('/proc/stat') ?: [] as $line) {
|
|
if (!preg_match('/^(cpu\d*)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)/', $line, $m)) continue;
|
|
$raw[$m[1]] = [(int)$m[2],(int)$m[3],(int)$m[4],(int)$m[5],(int)$m[6],(int)$m[7],(int)$m[8]];
|
|
}
|
|
|
|
$stateFile = '/tmp/vv_cpu_stat.json';
|
|
$prev = file_exists($stateFile) ? (json_decode(file_get_contents($stateFile), true) ?: []) : [];
|
|
file_put_contents($stateFile, json_encode($raw));
|
|
|
|
$usage = function(array $c, ?array $p): int {
|
|
if (!$p) return 0;
|
|
$dt = array_sum($c) - array_sum($p);
|
|
$di = ($c[3] + $c[4]) - ($p[3] + $p[4]);
|
|
return $dt > 0 ? max(0, min(100, (int)round((1 - $di / $dt) * 100))) : 0;
|
|
};
|
|
|
|
$overall = $usage($raw['cpu'] ?? [], $prev['cpu'] ?? null);
|
|
$cores = [];
|
|
foreach ($raw as $cpu => $c) {
|
|
if ($cpu === 'cpu') continue;
|
|
$num = (int)substr($cpu, 3);
|
|
$freqKhz = (int)@file_get_contents("/sys/devices/system/cpu/$cpu/cpufreq/scaling_cur_freq");
|
|
$maxKhz = (int)@file_get_contents("/sys/devices/system/cpu/$cpu/cpufreq/cpuinfo_max_freq");
|
|
$minKhz = (int)@file_get_contents("/sys/devices/system/cpu/$cpu/cpufreq/cpuinfo_min_freq");
|
|
$cores[] = [
|
|
'core' => $num,
|
|
'usage_pct' => $usage($c, $prev[$cpu] ?? null),
|
|
'freq_mhz' => $freqKhz > 0 ? (int)round($freqKhz / 1000) : 0,
|
|
'max_mhz' => $maxKhz > 0 ? (int)round($maxKhz / 1000) : 0,
|
|
'min_mhz' => $minKhz > 0 ? (int)round($minKhz / 1000) : 0,
|
|
];
|
|
}
|
|
usort($cores, fn($a, $b) => $a['core'] - $b['core']);
|
|
return ['overall' => $overall, 'cores' => $cores];
|
|
}
|
|
|
|
function vv_memory_breakdown(): array {
|
|
$mem = [];
|
|
foreach (file('/proc/meminfo') ?: [] as $line) {
|
|
if (preg_match('/^(\w+):\s+(\d+)/', $line, $m)) $mem[$m[1]] = (int)$m[2];
|
|
}
|
|
$totalKb = $mem['MemTotal'] ?? 0;
|
|
|
|
// ZFS ARC
|
|
$arcKb = 0;
|
|
foreach (@file('/proc/spl/kstat/zfs/arcstats') ?: [] as $line) {
|
|
if (preg_match('/^size\s+\d+\s+(\d+)/', $line, $m)) { $arcKb = (int)($m[1] / 1024); break; }
|
|
}
|
|
|
|
// Docker — cgroup v1 then v2
|
|
$dockerKb = 0;
|
|
$cgv1 = '/sys/fs/cgroup/memory/docker/memory.usage_in_bytes';
|
|
if (file_exists($cgv1)) {
|
|
$dockerKb = (int)((float)@file_get_contents($cgv1) / 1024);
|
|
} else {
|
|
foreach (glob('/sys/fs/cgroup/system.slice/docker-*.scope/memory.current') ?: [] as $f) {
|
|
$v = @file_get_contents($f);
|
|
if ($v !== false) $dockerKb += (int)((float)$v / 1024);
|
|
}
|
|
}
|
|
|
|
// VM (QEMU/KVM RSS)
|
|
$vmKb = 0;
|
|
foreach (preg_split('/\s+/', trim(shell_exec('ps -C qemu-system-x86_64 -o rss= 2>/dev/null') ?: '')) as $rss) {
|
|
if (is_numeric($rss) && $rss > 0) $vmKb += (int)$rss;
|
|
}
|
|
|
|
$freeKb = max(0, $mem['MemAvailable'] ?? 0);
|
|
$systemKb = max(0, $totalKb - $freeKb - $arcKb - $dockerKb - $vmKb);
|
|
|
|
// Top processes by RSS — group same-named procs, take top 5
|
|
$grouped = [];
|
|
$psOut = shell_exec("ps -eo comm,rss --sort=-rss 2>/dev/null | tail -n +2 | head -40") ?: '';
|
|
foreach (explode("\n", trim($psOut)) as $line) {
|
|
$parts = preg_split('/\s+/', trim($line), 2);
|
|
if (count($parts) === 2 && is_numeric($parts[1]) && (int)$parts[1] > 0)
|
|
$grouped[$parts[0]] = ($grouped[$parts[0]] ?? 0) + (int)$parts[1];
|
|
}
|
|
arsort($grouped);
|
|
$topProcs = [];
|
|
foreach (array_slice($grouped, 0, 3, true) as $name => $kb)
|
|
$topProcs[] = ['name' => $name, 'kb' => $kb];
|
|
|
|
return [
|
|
'total_kb' => $totalKb,
|
|
'system_kb' => $systemKb,
|
|
'vm_kb' => $vmKb,
|
|
'zfs_kb' => $arcKb,
|
|
'docker_kb' => $dockerKb,
|
|
'free_kb' => $freeKb,
|
|
'top_procs' => $topProcs,
|
|
];
|
|
}
|
|
|
|
function vv_df(string $path): array {
|
|
$out = shell_exec("df -BM --output=size,used,avail '$path' 2>/dev/null | tail -1");
|
|
if (!$out) return ['available' => false, 'path' => $path];
|
|
$parts = preg_split('/\s+/', trim($out));
|
|
return [
|
|
'available' => true,
|
|
'path' => $path,
|
|
'size_mb' => (int)$parts[0],
|
|
'used_mb' => (int)$parts[1],
|
|
'free_mb' => (int)$parts[2],
|
|
];
|
|
}
|
|
|
|
function vv_network_stats(): array {
|
|
$iface = trim(shell_exec("ip route show default 2>/dev/null | awk 'NR==1{print \$5}'") ?: '');
|
|
if (!$iface) {
|
|
$best = ''; $bestBytes = 0;
|
|
foreach (file('/proc/net/dev') ?: [] as $line) {
|
|
if (!preg_match('/^\s*(\w+):\s+(\d+)/', $line, $m) || $m[1] === 'lo') continue;
|
|
if ((int)$m[2] > $bestBytes) { $bestBytes = (int)$m[2]; $best = $m[1]; }
|
|
}
|
|
$iface = $best;
|
|
}
|
|
if (!$iface) return ['available' => false];
|
|
|
|
$rxBytes = $txBytes = 0;
|
|
foreach (file('/proc/net/dev') ?: [] as $line) {
|
|
if (!preg_match('/^\s*' . preg_quote($iface, '/') . ':\s+(.+)$/', $line, $m)) continue;
|
|
$parts = preg_split('/\s+/', trim($m[1]));
|
|
$rxBytes = (int)($parts[0] ?? 0);
|
|
$txBytes = (int)($parts[8] ?? 0);
|
|
break;
|
|
}
|
|
|
|
$stateFile = '/tmp/vv_net_stat.json';
|
|
$now = ['rx' => $rxBytes, 'tx' => $txBytes, 'ts' => microtime(true)];
|
|
$prev = file_exists($stateFile) ? (json_decode(file_get_contents($stateFile), true) ?: []) : [];
|
|
file_put_contents($stateFile, json_encode($now));
|
|
|
|
$rxRate = $txRate = 0;
|
|
if (!empty($prev['ts']) && ($dt = $now['ts'] - $prev['ts']) > 0.1) {
|
|
$rxRate = max(0, (int)(($rxBytes - ($prev['rx'] ?? $rxBytes)) / $dt));
|
|
$txRate = max(0, (int)(($txBytes - ($prev['tx'] ?? $txBytes)) / $dt));
|
|
}
|
|
|
|
$speedMbps = (int)@file_get_contents("/sys/class/net/$iface/speed");
|
|
|
|
// Local IP — use primary iface
|
|
$localIp = trim(shell_exec(
|
|
"ip -4 addr show " . escapeshellarg($iface) . " 2>/dev/null | awk '/inet /{print \$2}' | cut -d/ -f1 | head -1"
|
|
) ?: '');
|
|
|
|
// External IP — curl ifconfig.me, cached 5 min so we don't hammer it
|
|
$extIpCache = '/tmp/vv_ext_ip.cache';
|
|
$extIp = '';
|
|
if (file_exists($extIpCache) && (time() - filemtime($extIpCache)) < 300) {
|
|
$extIp = trim(file_get_contents($extIpCache) ?: '');
|
|
} else {
|
|
$fetched = trim(shell_exec('curl -sf --max-time 4 https://ifconfig.me 2>/dev/null') ?: '');
|
|
if (preg_match('/^\d+\.\d+\.\d+\.\d+$/', $fetched)) {
|
|
$extIp = $fetched;
|
|
file_put_contents($extIpCache, $extIp);
|
|
}
|
|
}
|
|
|
|
// Tailscale IP — use `tailscale ip` CLI (interface name varies: tailscale0, tailscale1, etc.)
|
|
$tsIp = trim(shell_exec('tailscale ip -4 2>/dev/null | head -1') ?: '');
|
|
|
|
return [
|
|
'available' => true,
|
|
'iface' => $iface,
|
|
'speed_mbps' => $speedMbps > 0 ? $speedMbps : null,
|
|
'rx_bps' => $rxRate,
|
|
'tx_bps' => $txRate,
|
|
'local_ip' => $localIp,
|
|
'ext_ip' => $extIp,
|
|
'ts_ip' => $tsIp,
|
|
];
|
|
}
|
|
|
|
function vv_fallback_state(): array {
|
|
// State file written by fallback.sh
|
|
$stateFile = '/tmp/fallback_state.db';
|
|
if (!file_exists($stateFile)) return ['state' => 'UNKNOWN'];
|
|
$raw = [];
|
|
foreach (file($stateFile) ?: [] as $line) {
|
|
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
|
$raw[trim($k)] = trim($v);
|
|
}
|
|
return [
|
|
'state' => $raw['state'] ?? 'UNKNOWN',
|
|
'failover_start' => $raw['failover_start'] ?? '0',
|
|
'tier2_started' => $raw['tier2_started'] ?? 'false',
|
|
'tier3_started' => $raw['tier3_started'] ?? 'false',
|
|
'tier4_started' => $raw['tier4_started'] ?? 'false',
|
|
];
|
|
}
|
|
|
|
function vv_transcode_sessions(): array {
|
|
$stateFile = '/tmp/transcode_state.db';
|
|
if (!file_exists($stateFile)) return ['available' => false];
|
|
|
|
$raw = [];
|
|
foreach (file($stateFile) ?: [] as $line) {
|
|
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
|
$raw[trim($k)] = trim($v);
|
|
}
|
|
|
|
$target = $raw['current_target'] ?? '';
|
|
$lastFlip = (int)($raw['last_flip_time'] ?? 0);
|
|
$flipCount = (int)($raw['flip_count_hour'] ?? 0);
|
|
$isRamdisk = str_contains($target, 'ramdisk');
|
|
|
|
// Count active session dirs in both known locations
|
|
$ramdiskPath = '/mnt/ramdisk_transcodes/transcoding-temp';
|
|
$ramSessions = count(glob("$ramdiskPath/*/", GLOB_ONLYDIR) ?: []);
|
|
|
|
// SSD path: look for any other transcoding-temp sibling
|
|
$ssdPath = '';
|
|
$ssdSessions = 0;
|
|
foreach (glob('/mnt/*/transcoding-temp/', GLOB_ONLYDIR) ?: [] as $p) {
|
|
if (!str_contains($p, 'ramdisk')) { $ssdPath = $p; break; }
|
|
}
|
|
if ($ssdPath) $ssdSessions = count(glob($ssdPath . '/*/', GLOB_ONLYDIR) ?: []);
|
|
|
|
// Ramdisk disk usage
|
|
$rd = vv_df('/mnt/ramdisk_transcodes');
|
|
|
|
return [
|
|
'available' => true,
|
|
'current_target' => $target,
|
|
'is_ramdisk' => $isRamdisk,
|
|
'flip_count_hour' => $flipCount,
|
|
'last_flip_time' => $lastFlip,
|
|
'last_flip_ago' => $lastFlip > 0 ? time() - $lastFlip : null,
|
|
'ram_sessions' => $ramSessions,
|
|
'ssd_sessions' => $ssdSessions,
|
|
'ramdisk' => $rd,
|
|
];
|
|
}
|