diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/include/monitor.php b/Plugin/usr/local/emhttp/plugins/varaverk/include/monitor.php index 8ed8fa3..d21fb92 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/include/monitor.php +++ b/Plugin/usr/local/emhttp/plugins/varaverk/include/monitor.php @@ -1,4 +1,6 @@ /dev/null'); + $out = shell_exec('nvidia-smi --query-gpu=name,memory.used,memory.total,utilization.gpu,temperature.gpu,power.draw,utilization.encoder,utilization.decoder --format=csv,noheader,nounits 2>/dev/null'); if (!$out) return ['available' => false]; $parts = array_map('trim', explode(',', $out)); + $power = is_numeric($parts[5] ?? '') ? round((float)$parts[5], 1) : null; return [ 'available' => true, 'name' => $parts[0] ?? '', @@ -35,6 +38,9 @@ function vv_gpu_stats(): array { 'memory_total' => (int)($parts[2] ?? 0), 'utilization' => (int)($parts[3] ?? 0), 'temperature' => (int)($parts[4] ?? 0), + 'power_w' => $power, + 'enc_pct' => (int)($parts[6] ?? 0), + 'dec_pct' => (int)($parts[7] ?? 0), ]; } @@ -54,26 +60,98 @@ function vv_gpu_processes(): array { } function vv_system_resources(): array { - // RAM $mem = []; foreach (file('/proc/meminfo') ?: [] as $line) { if (preg_match('/^(MemTotal|MemAvailable):\s+(\d+)/', $line, $m)) $mem[$m[1]] = (int)$m[2]; } + return [ + 'ram_total_mb' => (int)(($mem['MemTotal'] ?? 0) / 1024), + 'ram_free_mb' => (int)(($mem['MemAvailable'] ?? 0) / 1024), + 'cache' => vv_df('/mnt/cache'), + ]; +} - // CPU (1-second sample) - $cpu = (int)trim(shell_exec("top -bn1 | grep 'Cpu(s)' | awk '{print $2}' | cut -d. -f1") ?: '0'); +function vv_cpu_per_core(): array { + // Parse /proc/stat — [user, nice, system, idle, iowait, irq, softirq] + $raw = []; + foreach (file('/proc/stat') ?: [] as $line) { + if (!preg_match('/^(cpu\d*)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)/', $line, $m)) continue; + $raw[$m[1]] = [(int)$m[2],(int)$m[3],(int)$m[4],(int)$m[5],(int)$m[6],(int)$m[7],(int)$m[8]]; + } - // Disk — ramdisk + cache - $ramdisk = vv_df('/mnt/ramdisk_transcodes'); - $cache = vv_df('/mnt/cache'); + $stateFile = '/tmp/vv_cpu_stat.json'; + $prev = file_exists($stateFile) ? (json_decode(file_get_contents($stateFile), true) ?: []) : []; + file_put_contents($stateFile, json_encode($raw)); + + $usage = function(array $c, ?array $p): int { + if (!$p) return 0; + $dt = array_sum($c) - array_sum($p); + $di = ($c[3] + $c[4]) - ($p[3] + $p[4]); + return $dt > 0 ? max(0, min(100, (int)round((1 - $di / $dt) * 100))) : 0; + }; + + $overall = $usage($raw['cpu'] ?? [], $prev['cpu'] ?? null); + $cores = []; + foreach ($raw as $cpu => $c) { + if ($cpu === 'cpu') continue; + $num = (int)substr($cpu, 3); + $freqKhz = (int)@file_get_contents("/sys/devices/system/cpu/$cpu/cpufreq/scaling_cur_freq"); + $maxKhz = (int)@file_get_contents("/sys/devices/system/cpu/$cpu/cpufreq/cpuinfo_max_freq"); + $minKhz = (int)@file_get_contents("/sys/devices/system/cpu/$cpu/cpufreq/cpuinfo_min_freq"); + $cores[] = [ + 'core' => $num, + 'usage_pct' => $usage($c, $prev[$cpu] ?? null), + 'freq_mhz' => $freqKhz > 0 ? (int)round($freqKhz / 1000) : 0, + 'max_mhz' => $maxKhz > 0 ? (int)round($maxKhz / 1000) : 0, + 'min_mhz' => $minKhz > 0 ? (int)round($minKhz / 1000) : 0, + ]; + } + usort($cores, fn($a, $b) => $a['core'] - $b['core']); + return ['overall' => $overall, 'cores' => $cores]; +} + +function vv_memory_breakdown(): array { + $mem = []; + foreach (file('/proc/meminfo') ?: [] as $line) { + if (preg_match('/^(\w+):\s+(\d+)/', $line, $m)) $mem[$m[1]] = (int)$m[2]; + } + $totalKb = $mem['MemTotal'] ?? 0; + + // ZFS ARC + $arcKb = 0; + foreach (@file('/proc/spl/kstat/zfs/arcstats') ?: [] as $line) { + if (preg_match('/^size\s+\d+\s+(\d+)/', $line, $m)) { $arcKb = (int)($m[1] / 1024); break; } + } + + // Docker — cgroup v1 then v2 + $dockerKb = 0; + $cgv1 = '/sys/fs/cgroup/memory/docker/memory.usage_in_bytes'; + if (file_exists($cgv1)) { + $dockerKb = (int)((float)@file_get_contents($cgv1) / 1024); + } else { + foreach (glob('/sys/fs/cgroup/system.slice/docker-*.scope/memory.current') ?: [] as $f) { + $v = @file_get_contents($f); + if ($v !== false) $dockerKb += (int)((float)$v / 1024); + } + } + + // VM (QEMU/KVM RSS) + $vmKb = 0; + foreach (preg_split('/\s+/', trim(shell_exec('ps -C qemu-system-x86_64 -o rss= 2>/dev/null') ?: '')) as $rss) { + if (is_numeric($rss) && $rss > 0) $vmKb += (int)$rss; + } + + $freeKb = max(0, $mem['MemAvailable'] ?? 0); + $systemKb = max(0, $totalKb - $freeKb - $arcKb - $dockerKb - $vmKb); return [ - 'ram_total_mb' => (int)(($mem['MemTotal'] ?? 0) / 1024), - 'ram_free_mb' => (int)(($mem['MemAvailable'] ?? 0) / 1024), - 'cpu_percent' => $cpu, - 'ramdisk' => $ramdisk, - 'cache' => $cache, + 'total_kb' => $totalKb, + 'system_kb' => $systemKb, + 'vm_kb' => $vmKb, + 'zfs_kb' => $arcKb, + 'docker_kb' => $dockerKb, + 'free_kb' => $freeKb, ]; } @@ -90,6 +168,81 @@ function vv_df(string $path): array { ]; } +function vv_network_stats(): array { + $iface = trim(shell_exec("ip route show default 2>/dev/null | awk 'NR==1{print \$5}'") ?: ''); + if (!$iface) { + $best = ''; $bestBytes = 0; + foreach (file('/proc/net/dev') ?: [] as $line) { + if (!preg_match('/^\s*(\w+):\s+(\d+)/', $line, $m) || $m[1] === 'lo') continue; + if ((int)$m[2] > $bestBytes) { $bestBytes = (int)$m[2]; $best = $m[1]; } + } + $iface = $best; + } + if (!$iface) return ['available' => false]; + + $rxBytes = $txBytes = 0; + foreach (file('/proc/net/dev') ?: [] as $line) { + if (!preg_match('/^\s*' . preg_quote($iface, '/') . ':\s+(.+)$/', $line, $m)) continue; + $parts = preg_split('/\s+/', trim($m[1])); + $rxBytes = (int)($parts[0] ?? 0); + $txBytes = (int)($parts[8] ?? 0); + break; + } + + $stateFile = '/tmp/vv_net_stat.json'; + $now = ['rx' => $rxBytes, 'tx' => $txBytes, 'ts' => microtime(true)]; + $prev = file_exists($stateFile) ? (json_decode(file_get_contents($stateFile), true) ?: []) : []; + file_put_contents($stateFile, json_encode($now)); + + $rxRate = $txRate = 0; + if (!empty($prev['ts']) && ($dt = $now['ts'] - $prev['ts']) > 0.1) { + $rxRate = max(0, (int)(($rxBytes - ($prev['rx'] ?? $rxBytes)) / $dt)); + $txRate = max(0, (int)(($txBytes - ($prev['tx'] ?? $txBytes)) / $dt)); + } + + $speedMbps = (int)@file_get_contents("/sys/class/net/$iface/speed"); + + // Local IP — use the NIC defined in HOST*_SYS_WATCHDOG_NIC, fall back to primary iface + $vars = vv_conf_vars(); + $host = vv_detect_host(); + $prefix = $host !== 'unknown' ? strtoupper($host) . '_' : 'HOST1_'; + $nic = $vars[$prefix . 'SYS_WATCHDOG_NIC'] ?? $iface; + $localIp = trim(shell_exec( + "ip -4 addr show " . escapeshellarg($nic) . " 2>/dev/null | awk '/inet /{print \$2}' | cut -d/ -f1 | head -1" + ) ?: ''); + + // External IP — resolve DDNS domain from conf (no external HTTP; DDNS keeps it current) + $ddnsDomain = $vars[$prefix . 'NETWORK_WATCHDOG_DDNS_DOMAIN'] ?? ''; + $extIp = ''; + if ($ddnsDomain) { + $extIpCache = '/tmp/vv_ext_ip.cache'; + if (file_exists($extIpCache) && (time() - filemtime($extIpCache)) < 300) { + $extIp = trim(file_get_contents($extIpCache) ?: ''); + } else { + $resolved = trim(shell_exec( + "dig +short " . escapeshellarg($ddnsDomain) . " @1.1.1.1 2>/dev/null | grep -Eo '[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+' | head -1" + ) ?: ''); + if ($resolved) { $extIp = $resolved; file_put_contents($extIpCache, $extIp); } + } + } + + // Tailscale IP — read from tailscale0 interface + $tsIp = trim(shell_exec( + "ip -4 addr show tailscale0 2>/dev/null | awk '/inet /{print \$2}' | cut -d/ -f1 | head -1" + ) ?: ''); + + return [ + 'available' => true, + 'iface' => $iface, + 'speed_mbps' => $speedMbps > 0 ? $speedMbps : null, + 'rx_bps' => $rxRate, + 'tx_bps' => $txRate, + 'local_ip' => $localIp, + 'ext_ip' => $extIp, + 'ts_ip' => $tsIp, + ]; +} + function vv_fallback_state(): array { // State file written by fallback.sh $stateFile = '/tmp/fallback_state.db'; @@ -109,13 +262,44 @@ function vv_fallback_state(): array { } function vv_transcode_sessions(): array { - // Read from transcode state file written by transcode_manager.sh $stateFile = '/tmp/transcode_state.db'; - if (!file_exists($stateFile)) return []; + if (!file_exists($stateFile)) return ['available' => false]; + $raw = []; foreach (file($stateFile) ?: [] as $line) { [$k, $v] = array_pad(explode('=', trim($line), 2), 2, ''); $raw[trim($k)] = trim($v); } - return $raw; + + $target = $raw['current_target'] ?? ''; + $lastFlip = (int)($raw['last_flip_time'] ?? 0); + $flipCount = (int)($raw['flip_count_hour'] ?? 0); + $isRamdisk = str_contains($target, 'ramdisk'); + + // Count active session dirs in both known locations + $ramdiskPath = '/mnt/ramdisk_transcodes/transcoding-temp'; + $ramSessions = count(glob("$ramdiskPath/*/", GLOB_ONLYDIR) ?: []); + + // SSD path: look for any other transcoding-temp sibling + $ssdPath = ''; + $ssdSessions = 0; + foreach (glob('/mnt/*/transcoding-temp/', GLOB_ONLYDIR) ?: [] as $p) { + if (!str_contains($p, 'ramdisk')) { $ssdPath = $p; break; } + } + if ($ssdPath) $ssdSessions = count(glob($ssdPath . '/*/', GLOB_ONLYDIR) ?: []); + + // Ramdisk disk usage + $rd = vv_df('/mnt/ramdisk_transcodes'); + + return [ + 'available' => true, + 'current_target' => $target, + 'is_ramdisk' => $isRamdisk, + 'flip_count_hour' => $flipCount, + 'last_flip_time' => $lastFlip, + 'last_flip_ago' => $lastFlip > 0 ? time() - $lastFlip : null, + 'ram_sessions' => $ramSessions, + 'ssd_sessions' => $ssdSessions, + 'ramdisk' => $rd, + ]; }