1223 lines
56 KiB
PHP
1223 lines
56 KiB
PHP
<?php
|
||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||
// PURPOSE
|
||
// The system-metrics library. Everything the monitor page shows about this machine —
|
||
// CPU per core, memory breakdown, GPUs, disks and pools, network, UPS, parity, VMs,
|
||
// containers, transcodes — plus a roll-up of the same for remote nodes.
|
||
//
|
||
// DESIGN PRINCIPLES
|
||
// Prefer the Unraid API, fall back to reading the system directly.
|
||
// vv_api_data() is tried first; when it is unavailable each metric has a local path
|
||
// (/proc, /sys, emhttp ini files, shell tools). The API going away degrades detail,
|
||
// never the page.
|
||
//
|
||
// Remote stats read every host*.conf, not just this host's.
|
||
// A partner's API key lives in the partner's own conf. vv_remote_hosts_stats() globs
|
||
// CONF_DIR for all host*.conf and merges what it finds, because sparse checkout means
|
||
// the partner's file arrives through the conf cache rather than from git.
|
||
//
|
||
// Background cache first, live call second.
|
||
// Remote payloads written by remote_arr_cache_writer.sh (every 2h) are used when
|
||
// present; otherwise a live call runs behind a 30s inline cache. The expensive path is
|
||
// the exception, not the default.
|
||
//
|
||
// Reports raw numbers, applies no policy.
|
||
// Thresholds, alerting and remediation belong to the watchdogs. This file answers
|
||
// "what is the value" and nothing else.
|
||
//
|
||
// OPERATIONAL SAFEGUARDS
|
||
// Every read degrades to empty, never fatal.
|
||
// Filesystem reads use @ with a ?: fallback and every shell_exec redirects stderr.
|
||
// A missing GPU, absent UPS, or unreadable sysfs node yields [] and the corresponding
|
||
// card simply does not render. One missing subsystem cannot blank the whole page.
|
||
//
|
||
// Absent tooling is a normal outcome.
|
||
// No nvidia-smi means no GPU section — not an error. The page is built to be correct
|
||
// on hardware that lacks any given subsystem.
|
||
//
|
||
// Missing state files return an explicit unavailable flag.
|
||
// vv_transcode_sessions() returns ['available' => false] when transcode_state.db does
|
||
// not exist, so the caller can distinguish "not running" from "zero sessions".
|
||
//
|
||
// External IP lookups are cached and time-boxed.
|
||
// curl runs with --max-time and the result is cached 300s, so a slow or unreachable
|
||
// endpoint cannot stall a page render.
|
||
//
|
||
// Read-only throughout. Nothing here starts, stops, or reconfigures anything.
|
||
//
|
||
// EXPORTS
|
||
// System vv_system_info(), vv_system_resources(), vv_cpu_per_core(), vv_memory_breakdown()
|
||
// Storage vv_df(), vv_storage_pools(), vv_array_disks(), vv_disk_io_rates(),
|
||
// vv_disk_thresholds(), vv_disk_entry(), vv_parity_status()
|
||
// Hardware vv_gpu_stats(), vv_gpu_stats_all(), vv_gpu_processes(), vv_ups_stats()
|
||
// Containers vv_docker_containers(), vv_docker_stopped()
|
||
// Network vv_network_stats()
|
||
// Remote vv_remote_hosts_stats()
|
||
// Misc vv_transcode_sessions(), vv_log_tail(), vv_parse_bash_array()
|
||
//
|
||
// CONFIGURATION
|
||
// STATE_DIR transcode_state.db lives here
|
||
// HOST*_UNRAID_API_KEY per-host, read from every host*.conf for remote metrics
|
||
// VV_CACHE_DIR ext_ip (300s) and monitor_remote_<host> (written externally)
|
||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||
|
||
require_once __DIR__ . '/config.php';
|
||
require_once __DIR__ . '/unraid_api.php';
|
||
|
||
// Timestamp out of the container restart log, whose second field is a formatted local date
|
||
// ("2026-08-02 13:15:07") rather than an epoch — docker_watchdog.sh writes it that way because
|
||
// its own rolling-window trim compares the strings lexically in awk.
|
||
//
|
||
// Lives here rather than in watchdog.php because include/monitor.php parses the same file for
|
||
// the dashboard card and does not include watchdog.php. Both readers previously cast the field
|
||
// with (int), which stops at the first non-digit and returned 2026 for every line ever written —
|
||
// below any cutoff, so both restart lists were permanently empty and looked exactly like
|
||
// "nothing has restarted".
|
||
//
|
||
// Accepts an epoch too, so that changing the writer later does not require changing the readers.
|
||
function vv_wd_restart_ts(string $raw): int {
|
||
$raw = trim($raw);
|
||
if (ctype_digit($raw)) return (int)$raw;
|
||
return (int)(strtotime($raw) ?: 0);
|
||
}
|
||
|
||
function vv_system_info(): array {
|
||
// ── Shared local reads (always needed regardless of API) ──────────────────
|
||
$ident = @parse_ini_file('/boot/config/ident.cfg') ?: [];
|
||
$var = [];
|
||
foreach (@file('/var/local/emhttp/var.ini') ?: [] as $line) {
|
||
if (preg_match('/^(\w+)="([^"]*)"/', $line, $m)) $var[$m[1]] = $m[2];
|
||
}
|
||
$version = trim(@file_get_contents('/etc/unraid-version') ?: '');
|
||
|
||
// ── API path ──────────────────────────────────────────────────────────────
|
||
$api = vv_api_data();
|
||
if ($api) {
|
||
$os = $api['info']['os'] ?? [];
|
||
$cpu = $api['info']['cpu'] ?? [];
|
||
|
||
// uptime is a String in this schema — try numeric (seconds) first, else fall back to /proc/uptime
|
||
$uptimeRaw = $os['uptime'] ?? '';
|
||
if (is_numeric($uptimeRaw)) {
|
||
$uptimeSec = (int)$uptimeRaw;
|
||
} else {
|
||
$uptimeSec = (int)explode(' ', @file_get_contents('/proc/uptime') ?: '0')[0];
|
||
}
|
||
$uptime = vv_format_uptime($uptimeSec);
|
||
|
||
$load = sys_getloadavg();
|
||
return [
|
||
'name' => $os['hostname'] ?? ($ident['NAME'] ?? gethostname()),
|
||
'comment' => $ident['COMMENT'] ?? '',
|
||
'timezone' => $ident['timeZone'] ?? 'UTC',
|
||
'cpu_model' => $cpu['brand'] ?? ($ident['SYS_MODEL'] ?? ''),
|
||
'cpu_threads' => (int)($cpu['threads'] ?? 0),
|
||
'cpu_cores' => (int)($cpu['cores'] ?? 0),
|
||
'reg_type' => 'Unraid OS ' . ($var['regTy'] ?? ''),
|
||
'reg_to' => $var['regTo'] ?? '',
|
||
'uptime' => $uptime,
|
||
'uptime_sec' => $uptimeSec,
|
||
'array_state' => strtoupper($api['array']['state'] ?? $var['mdState'] ?? 'UNKNOWN'),
|
||
'version' => trim($os['release'] ?? '') ?: $version,
|
||
'load_avg' => $load ? [round($load[0], 2), round($load[1], 2), round($load[2], 2)] : null,
|
||
];
|
||
}
|
||
|
||
// ── Local fallback ────────────────────────────────────────────────────────
|
||
vv_api_record_fallback('system_info');
|
||
$cpuModel = '';
|
||
foreach (@file('/proc/cpuinfo') ?: [] as $line) {
|
||
if (preg_match('/^model name\s*:\s*(.+)/', $line, $m)) { $cpuModel = trim($m[1]); break; }
|
||
}
|
||
$uptimeSec = (int)explode(' ', @file_get_contents('/proc/uptime') ?: '0')[0];
|
||
$uptime = vv_format_uptime($uptimeSec);
|
||
|
||
$load = sys_getloadavg();
|
||
return [
|
||
'name' => $ident['NAME'] ?? gethostname(),
|
||
'comment' => $ident['COMMENT'] ?? '',
|
||
'timezone' => $ident['timeZone'] ?? 'UTC',
|
||
'cpu_model' => $ident['SYS_MODEL'] ?? $cpuModel,
|
||
'cpu_threads' => 0,
|
||
'cpu_cores' => 0,
|
||
'reg_type' => 'Unraid OS ' . ($var['regTy'] ?? ''),
|
||
'reg_to' => $var['regTo'] ?? '',
|
||
'uptime' => $uptime,
|
||
'uptime_sec' => $uptimeSec,
|
||
'array_state' => $var['mdState'] ?? 'UNKNOWN',
|
||
'version' => $version,
|
||
'load_avg' => $load ? [round($load[0], 2), round($load[1], 2), round($load[2], 2)] : null,
|
||
];
|
||
}
|
||
|
||
function vv_docker_containers(): array {
|
||
$out = shell_exec('docker ps --format \'{"name":"{{.Names}}","status":"{{.Status}}","image":"{{.Image}}"}\' 2>/dev/null');
|
||
$containers = [];
|
||
foreach (explode("\n", trim($out ?? '')) as $line) {
|
||
if (!$line) continue;
|
||
$c = json_decode($line, true);
|
||
if ($c) $containers[] = $c;
|
||
}
|
||
return $containers;
|
||
}
|
||
|
||
function vv_docker_stopped(): array {
|
||
$out = shell_exec('docker ps -a --filter "status=exited" --filter "status=created" --format \'{"name":"{{.Names}}","status":"{{.Status}}"}\' 2>/dev/null');
|
||
$containers = [];
|
||
foreach (explode("\n", trim($out ?? '')) as $line) {
|
||
if (!$line) continue;
|
||
$c = json_decode($line, true);
|
||
if ($c) $containers[] = $c;
|
||
}
|
||
return $containers;
|
||
}
|
||
|
||
// Every installed GPU, one entry per card. Parsed line by line — nvidia-smi emits one row
|
||
// per GPU, so splitting the whole output on commas (as this once did) runs the rows together
|
||
// and only ever describes GPU 0.
|
||
function vv_gpu_stats_all(): array {
|
||
// No early return when nvidia-smi is absent: a box with no discrete card still has to reach
|
||
// the integrated-graphics check below. Returning here is what kept HOST2 reporting no GPU
|
||
// even after that check existed.
|
||
$out = shell_exec('nvidia-smi --query-gpu=index,uuid,name,memory.used,memory.total,utilization.gpu,temperature.gpu,power.draw,utilization.encoder,utilization.decoder --format=csv,noheader,nounits 2>/dev/null');
|
||
$gpus = [];
|
||
foreach (explode("\n", trim((string)$out)) as $line) {
|
||
if (!$line) continue;
|
||
$p = array_map('trim', explode(',', $line));
|
||
if (count($p) < 10) continue;
|
||
$gpus[] = [
|
||
'available' => true,
|
||
'index' => (int)$p[0],
|
||
'uuid' => $p[1],
|
||
'name' => $p[2],
|
||
'memory_used' => (int)$p[3],
|
||
'memory_total' => (int)$p[4],
|
||
'utilization' => (int)$p[5],
|
||
'temperature' => (int)$p[6],
|
||
'power_w' => is_numeric($p[7]) ? round((float)$p[7], 1) : null,
|
||
'enc_pct' => (int)$p[8],
|
||
'dec_pct' => (int)$p[9],
|
||
'vendor' => 'nvidia',
|
||
];
|
||
}
|
||
// Integrated graphics after the discrete cards, numbered on from them. A box with both shows
|
||
// both; a box with only an iGPU stops reporting that it has no GPU at all.
|
||
return array_merge($gpus, vv_igpu_stats_all(count($gpus)));
|
||
}
|
||
|
||
// Intel integrated graphics, which nvidia-smi cannot see and which therefore rendered as "no GPU
|
||
// detected" on any box without a discrete card — HOST2 has UHD Graphics 730 and showed nothing.
|
||
//
|
||
// An iGPU is not a small discrete card and the difference is not cosmetic. It has no VRAM (it
|
||
// shares system memory), no temperature sensor of its own (it is inside the CPU package, which
|
||
// vv_cpu_temp() already reports), and no encode/decode split — Intel publishes per-engine busy
|
||
// figures instead: Render/3D, Blitter, Video, VideoEnhance. Those fields are returned null rather
|
||
// than zero, because 0 MB of VRAM and 0°C are claims, and "this part does not have one" is not.
|
||
//
|
||
// Utilisation comes from rc6, the idle residency percentage: busy is what is left of it. That is
|
||
// the whole-GPU number; the engine breakdown is carried alongside because on a media server the
|
||
// Video engine is the one worth watching — it is what a hardware transcode actually uses.
|
||
//
|
||
// Sampled, not read: intel_gpu_top measures over an interval, so this costs roughly the sample
|
||
// window. Memoised per request and bounded by timeout, because the cache writer calls it once a
|
||
// minute and nothing should be able to hang that.
|
||
function vv_igpu_stats_all(int $startIndex = 0): array {
|
||
static $cache = null;
|
||
if ($cache !== null) return $cache;
|
||
$cache = [];
|
||
|
||
foreach ((array)@glob('/sys/class/drm/card[0-9]*') as $card) {
|
||
if (trim((string)@file_get_contents("$card/device/vendor")) !== '0x8086') continue;
|
||
|
||
$addr = basename((string)@readlink("$card/device"));
|
||
if ($addr === '') continue;
|
||
|
||
// Marketing name where lspci publishes one: "Alder Lake-S GT1 [UHD Graphics 730]" is
|
||
// better known as UHD Graphics 730, and the bracketed half is the half people recognise.
|
||
$desc = trim((string)shell_exec('lspci -s ' . escapeshellarg($addr) . ' 2>/dev/null'));
|
||
$name = 'Intel integrated graphics';
|
||
if (preg_match('/\[([^\]]+)\]/', $desc, $m)) $name = 'Intel ' . trim($m[1]);
|
||
elseif (preg_match('/controller:\s*(.+?)(?:\s*\(rev|$)/', $desc, $m)) $name = trim($m[1]);
|
||
|
||
$entry = [
|
||
'available' => true,
|
||
'vendor' => 'intel',
|
||
'index' => $startIndex + count($cache),
|
||
// Prefixed so a discrete card's processes can never be attributed to this one on a
|
||
// box that has both.
|
||
'uuid' => 'intel:' . $addr,
|
||
'name' => $name,
|
||
'memory_used' => null, // shared system memory — there is no separate pool
|
||
'memory_total' => null,
|
||
'utilization' => null,
|
||
'temperature' => null, // no die sensor of its own; it is in the CPU package
|
||
'power_w' => null,
|
||
'package_w' => null,
|
||
'enc_pct' => null, // Intel reports engines, not an encode/decode split
|
||
'dec_pct' => null,
|
||
'engines' => [],
|
||
];
|
||
|
||
// The card still renders without this — name and "no sample" beats no card at all, which
|
||
// is what the box showed before.
|
||
$sample = vv_igpu_sample();
|
||
if ($sample !== null) {
|
||
$rc6 = $sample['rc6']['value'] ?? null;
|
||
if (is_numeric($rc6)) $entry['utilization'] = max(0, min(100, (int)round(100 - (float)$rc6)));
|
||
|
||
$gpuW = $sample['power']['GPU'] ?? null;
|
||
$pkgW = $sample['power']['Package'] ?? null;
|
||
// Alder Lake reports 0.00 for the GPU rail. Reported as null rather than as a
|
||
// confident zero watts, with package power carried separately and labelled as such.
|
||
if (is_numeric($gpuW) && (float)$gpuW > 0) $entry['power_w'] = round((float)$gpuW, 1);
|
||
if (is_numeric($pkgW)) $entry['package_w'] = round((float)$pkgW, 1);
|
||
|
||
foreach ((array)($sample['engines'] ?? []) as $engine => $vals) {
|
||
if (!is_array($vals) || !isset($vals['busy'])) continue;
|
||
$entry['engines'][$engine] = round((float)$vals['busy'], 1);
|
||
}
|
||
}
|
||
|
||
$cache[] = $entry;
|
||
}
|
||
return $cache;
|
||
}
|
||
|
||
// One sample from intel_gpu_top, decoded, or null.
|
||
//
|
||
// -J streams an unterminated JSON array: a bare "[" and then one object per interval, forever.
|
||
// Neither half is valid JSON on its own, so the first object is cut out by hand — from its opening
|
||
// brace to the first closing brace in column 1 — and decoded alone.
|
||
//
|
||
// timeout, and a short window: this runs inside the once-a-minute cache write and a sampler that
|
||
// never returns would take the whole payload with it.
|
||
function vv_igpu_sample(): ?array {
|
||
static $sample = null;
|
||
static $tried = false;
|
||
if ($tried) return $sample;
|
||
$tried = true;
|
||
|
||
if (trim((string)shell_exec('command -v intel_gpu_top 2>/dev/null')) === '') return null;
|
||
|
||
// stderr silenced across the whole pipeline, not just the sampler. Cutting the stream short
|
||
// makes sed exit while intel_gpu_top is still writing, and it says so — "couldn't flush
|
||
// stdout: Broken pipe", once a minute, into a log the repair sweep reads as a real warning.
|
||
$raw = shell_exec('{ timeout 4 intel_gpu_top -J -s 600 | sed -n "/^{/,/^}/p" | sed "/^}/q"; } 2>/dev/null');
|
||
if (!$raw) return null;
|
||
|
||
$decoded = json_decode(trim($raw), true);
|
||
$sample = is_array($decoded) ? $decoded : null;
|
||
return $sample;
|
||
}
|
||
|
||
// Kept for the existing single-GPU consumers — same shape as before, always GPU 0.
|
||
function vv_gpu_stats(): array {
|
||
$gpus = vv_gpu_stats_all();
|
||
return $gpus[0] ?? ['available' => false];
|
||
}
|
||
|
||
// gpu_uuid is included so each process can be attributed to the card it is actually running
|
||
// on. Without it a two-GPU box shows every process under every card.
|
||
function vv_gpu_processes(): array {
|
||
$out = shell_exec('nvidia-smi --query-compute-apps=gpu_uuid,pid,used_gpu_memory,name --format=csv,noheader,nounits 2>/dev/null');
|
||
$procs = [];
|
||
foreach (explode("\n", trim($out ?? '')) as $line) {
|
||
if (!$line) continue;
|
||
$parts = array_map('trim', explode(',', $line));
|
||
$procs[] = [
|
||
'gpu_uuid' => $parts[0] ?? '',
|
||
'pid' => $parts[1] ?? '',
|
||
'memory_mb' => $parts[2] ?? '',
|
||
'name' => $parts[3] ?? '',
|
||
];
|
||
}
|
||
return $procs;
|
||
}
|
||
|
||
function vv_system_resources(): array {
|
||
$mem = [];
|
||
foreach (file('/proc/meminfo') ?: [] as $line) {
|
||
if (preg_match('/^(MemTotal|MemAvailable):\s+(\d+)/', $line, $m))
|
||
$mem[$m[1]] = (int)$m[2];
|
||
}
|
||
return [
|
||
'ram_total_mb' => (int)(($mem['MemTotal'] ?? 0) / 1024),
|
||
'ram_free_mb' => (int)(($mem['MemAvailable'] ?? 0) / 1024),
|
||
'cache' => vv_df('/mnt/cache'),
|
||
];
|
||
}
|
||
|
||
function vv_cpu_per_core(): array {
|
||
// Parse /proc/stat — [user, nice, system, idle, iowait, irq, softirq]
|
||
$raw = [];
|
||
foreach (file('/proc/stat') ?: [] as $line) {
|
||
if (!preg_match('/^(cpu\d*)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)/', $line, $m)) continue;
|
||
$raw[$m[1]] = [(int)$m[2],(int)$m[3],(int)$m[4],(int)$m[5],(int)$m[6],(int)$m[7],(int)$m[8]];
|
||
}
|
||
|
||
$stateFile = VV_CACHE_DIR . '/vv_cpu_stat.json';
|
||
$prev = file_exists($stateFile) ? (json_decode(file_get_contents($stateFile), true) ?: []) : [];
|
||
// Atomic write — concurrent fast/slow polls read a consistent snapshot
|
||
$tmp = $stateFile . '.tmp';
|
||
file_put_contents($tmp, json_encode($raw));
|
||
rename($tmp, $stateFile);
|
||
|
||
$usage = function(array $c, ?array $p): int {
|
||
if (!$p) return 0;
|
||
$dt = array_sum($c) - array_sum($p);
|
||
$di = ($c[3] + $c[4]) - ($p[3] + $p[4]);
|
||
return $dt > 0 ? max(0, min(100, (int)round((1 - $di / $dt) * 100))) : 0;
|
||
};
|
||
|
||
$overall = $usage($raw['cpu'] ?? [], $prev['cpu'] ?? null);
|
||
$cores = [];
|
||
foreach ($raw as $cpu => $c) {
|
||
if ($cpu === 'cpu') continue;
|
||
$num = (int)substr($cpu, 3);
|
||
$freqKhz = (int)@file_get_contents("/sys/devices/system/cpu/$cpu/cpufreq/scaling_cur_freq");
|
||
$maxKhz = (int)@file_get_contents("/sys/devices/system/cpu/$cpu/cpufreq/cpuinfo_max_freq");
|
||
$minKhz = (int)@file_get_contents("/sys/devices/system/cpu/$cpu/cpufreq/cpuinfo_min_freq");
|
||
$cores[] = [
|
||
'core' => $num,
|
||
'usage_pct' => $usage($c, $prev[$cpu] ?? null),
|
||
'freq_mhz' => $freqKhz > 0 ? (int)round($freqKhz / 1000) : 0,
|
||
'max_mhz' => $maxKhz > 0 ? (int)round($maxKhz / 1000) : 0,
|
||
'min_mhz' => $minKhz > 0 ? (int)round($minKhz / 1000) : 0,
|
||
];
|
||
}
|
||
usort($cores, fn($a, $b) => $a['core'] - $b['core']);
|
||
return ['overall' => $overall, 'cores' => $cores];
|
||
}
|
||
|
||
function vv_memory_breakdown(): array {
|
||
$mem = [];
|
||
foreach (file('/proc/meminfo') ?: [] as $line) {
|
||
if (preg_match('/^(\w+):\s+(\d+)/', $line, $m)) $mem[$m[1]] = (int)$m[2];
|
||
}
|
||
$totalKb = $mem['MemTotal'] ?? 0;
|
||
|
||
// ZFS ARC
|
||
$arcKb = 0;
|
||
foreach (@file('/proc/spl/kstat/zfs/arcstats') ?: [] as $line) {
|
||
if (preg_match('/^size\s+\d+\s+(\d+)/', $line, $m)) { $arcKb = (int)($m[1] / 1024); break; }
|
||
}
|
||
|
||
// Docker — sum docker stats used memory per container (matches Unraid dashboard)
|
||
$dockerKb = 0;
|
||
$dsOut = shell_exec("docker stats --no-stream --format '{{.MemUsage}}' 2>/dev/null") ?: '';
|
||
foreach (explode("\n", trim($dsOut)) as $line) {
|
||
if (!preg_match('/^([0-9.]+)(GiB|MiB|KiB|B)\s*\//', trim($line), $m)) continue;
|
||
$val = (float)$m[1];
|
||
$dockerKb += match($m[2]) {
|
||
'GiB' => (int)($val * 1048576),
|
||
'MiB' => (int)($val * 1024),
|
||
'KiB' => (int)$val,
|
||
default => (int)($val / 1024),
|
||
};
|
||
}
|
||
|
||
// VM (QEMU/KVM RSS)
|
||
$vmKb = 0;
|
||
foreach (preg_split('/\s+/', trim(shell_exec('ps -C qemu-system-x86_64 -o rss= 2>/dev/null') ?: '')) as $rss) {
|
||
if (is_numeric($rss) && $rss > 0) $vmKb += (int)$rss;
|
||
}
|
||
|
||
$freeKb = max(0, $mem['MemAvailable'] ?? 0);
|
||
$systemKb = max(0, $totalKb - $freeKb - $arcKb - $dockerKb - $vmKb);
|
||
|
||
// Top processes by RSS — group same-named procs, take top 5
|
||
$grouped = [];
|
||
// stderr silenced for the pipeline, not just for ps: head -40 exits on the fortieth line and
|
||
// leaves tail writing into a closed pipe, which tail reports. Harmless, but it went to stderr
|
||
// once a minute from the cache writer on both hosts, and the repair sweep reads that log.
|
||
$psOut = shell_exec("{ ps -eo comm,rss --sort=-rss | tail -n +2 | head -40; } 2>/dev/null") ?: '';
|
||
foreach (explode("\n", trim($psOut)) as $line) {
|
||
$parts = preg_split('/\s+/', trim($line), 2);
|
||
if (count($parts) === 2 && is_numeric($parts[1]) && (int)$parts[1] > 0)
|
||
$grouped[$parts[0]] = ($grouped[$parts[0]] ?? 0) + (int)$parts[1];
|
||
}
|
||
arsort($grouped);
|
||
$topProcs = [];
|
||
foreach (array_slice($grouped, 0, 3, true) as $name => $kb)
|
||
$topProcs[] = ['name' => $name, 'kb' => $kb];
|
||
|
||
// Swap — from API metrics when available, else /proc/meminfo
|
||
$swapTotalKb = 0; $swapUsedKb = 0;
|
||
$apiMem = vv_api_data()['metrics']['memory'] ?? [];
|
||
if (!empty($apiMem['swapTotal'])) {
|
||
$swapTotalKb = (int)(((float)$apiMem['swapTotal']) / 1024);
|
||
$swapUsedKb = (int)(((float)$apiMem['swapUsed']) / 1024);
|
||
} else {
|
||
$swapTotalKb = $mem['SwapTotal'] ?? 0;
|
||
$swapUsedKb = ($mem['SwapTotal'] ?? 0) - ($mem['SwapFree'] ?? 0);
|
||
}
|
||
|
||
return [
|
||
'total_kb' => $totalKb,
|
||
'system_kb' => $systemKb,
|
||
'vm_kb' => $vmKb,
|
||
'zfs_kb' => $arcKb,
|
||
'docker_kb' => $dockerKb,
|
||
'free_kb' => $freeKb,
|
||
'swap_total_kb' => $swapTotalKb,
|
||
'swap_used_kb' => $swapUsedKb,
|
||
'top_procs' => $topProcs,
|
||
];
|
||
}
|
||
|
||
function vv_df(string $path): array {
|
||
$out = shell_exec("df -BM --output=size,used,avail '$path' 2>/dev/null | tail -1");
|
||
if (!$out) return ['available' => false, 'path' => $path];
|
||
$parts = preg_split('/\s+/', trim($out));
|
||
return [
|
||
'available' => true,
|
||
'path' => $path,
|
||
'size_mb' => (int)$parts[0],
|
||
'used_mb' => (int)$parts[1],
|
||
'free_mb' => (int)$parts[2],
|
||
];
|
||
}
|
||
|
||
function vv_network_stats(): array {
|
||
$iface = trim(shell_exec("ip route show default 2>/dev/null | awk 'NR==1{print \$5}'") ?: '');
|
||
if (!$iface) {
|
||
$best = ''; $bestBytes = 0;
|
||
foreach (file('/proc/net/dev') ?: [] as $line) {
|
||
if (!preg_match('/^\s*(\w+):\s+(\d+)/', $line, $m) || $m[1] === 'lo') continue;
|
||
if ((int)$m[2] > $bestBytes) { $bestBytes = (int)$m[2]; $best = $m[1]; }
|
||
}
|
||
$iface = $best;
|
||
}
|
||
if (!$iface) return ['available' => false];
|
||
|
||
$rxBytes = $txBytes = 0;
|
||
foreach (file('/proc/net/dev') ?: [] as $line) {
|
||
if (!preg_match('/^\s*' . preg_quote($iface, '/') . ':\s+(.+)$/', $line, $m)) continue;
|
||
$parts = preg_split('/\s+/', trim($m[1]));
|
||
$rxBytes = (int)($parts[0] ?? 0);
|
||
$txBytes = (int)($parts[8] ?? 0);
|
||
break;
|
||
}
|
||
|
||
$stateFile = VV_CACHE_DIR . '/vv_net_stat.json';
|
||
$now = ['rx' => $rxBytes, 'tx' => $txBytes, 'ts' => microtime(true)];
|
||
$prev = file_exists($stateFile) ? (json_decode(file_get_contents($stateFile), true) ?: []) : [];
|
||
$tmp = $stateFile . '.tmp';
|
||
file_put_contents($tmp, json_encode($now));
|
||
rename($tmp, $stateFile);
|
||
|
||
// ×8, because /proc/net/dev counts octets and everything downstream of here says bits.
|
||
//
|
||
// The field is named rx_bps, vvFmtBps() renders it as Kb/s, Mb/s and Gb/s, and the Monitor
|
||
// card prints it directly beneath the NIC's link speed — which comes from
|
||
// /sys/class/net/*/speed and genuinely is megabits. So the one number on that card you would
|
||
// read against the link was understating traffic by a factor of eight: 811 KB/s of real
|
||
// traffic displayed as "0.8 Mb/s" next to a 10 Gb/s link, when it was 6.5 Mb/s.
|
||
//
|
||
// Converted at the source rather than in the formatter so the field name stops lying. Both
|
||
// consumers are the Monitor card's rate readout and its history graph, and both are relative
|
||
// or bit-labelled, so neither changes meaning — only magnitude, to the correct one.
|
||
$rxRate = $txRate = 0;
|
||
if (!empty($prev['ts']) && ($dt = $now['ts'] - $prev['ts']) > 0.1) {
|
||
$rxRate = max(0, (int)(($rxBytes - ($prev['rx'] ?? $rxBytes)) * 8 / $dt));
|
||
$txRate = max(0, (int)(($txBytes - ($prev['tx'] ?? $txBytes)) * 8 / $dt));
|
||
}
|
||
|
||
$speedMbps = (int)@file_get_contents("/sys/class/net/$iface/speed");
|
||
|
||
// Local IP — use primary iface
|
||
$localIp = trim(shell_exec(
|
||
"ip -4 addr show " . escapeshellarg($iface) . " 2>/dev/null | awk '/inet /{print \$2}' | cut -d/ -f1 | head -1"
|
||
) ?: '');
|
||
|
||
// External IP — curl ifconfig.me, cached 5 min so we don't hammer it
|
||
$extIp = '';
|
||
$extData = vv_cache_read('ext_ip', 300);
|
||
if ($extData) {
|
||
$extIp = $extData['ip'] ?? '';
|
||
} else {
|
||
$fetched = trim(shell_exec('curl -sf --max-time 4 https://ifconfig.me 2>/dev/null') ?: '');
|
||
if (preg_match('/^\d+\.\d+\.\d+\.\d+$/', $fetched)) {
|
||
$extIp = $fetched;
|
||
vv_cache_write('ext_ip', ['ip' => $extIp]);
|
||
}
|
||
}
|
||
|
||
// Tailscale IP — use `tailscale ip` CLI (interface name varies: tailscale0, tailscale1, etc.)
|
||
// Bounded like every other tailscale lookup in the tree. vv_network_stats() is served by
|
||
// api/monitor_fast.php, which the Monitor page polls once a second; an unbounded call here
|
||
// makes a slow tailscale silently drop that live cadence to whatever the CLI takes.
|
||
$tsIp = trim(shell_exec('timeout 2 tailscale ip -4 2>/dev/null | head -1') ?: '');
|
||
|
||
return [
|
||
'available' => true,
|
||
'iface' => $iface,
|
||
'speed_mbps' => $speedMbps > 0 ? $speedMbps : null,
|
||
'rx_bps' => $rxRate,
|
||
'tx_bps' => $txRate,
|
||
'local_ip' => $localIp,
|
||
'ext_ip' => $extIp,
|
||
'ts_ip' => $tsIp,
|
||
];
|
||
}
|
||
|
||
function vv_disk_entry(array $d, string $key, string $role = 'data'): ?array {
|
||
$name = $d['name'] ?? $key;
|
||
$isParity = $role === 'parity';
|
||
$mounted = ($d['fsStatus'] ?? '') === 'Mounted';
|
||
// Parity has no filesystem — use raw size only
|
||
$size_kb = (int)($isParity ? ($d['size'] ?? 0) : ($mounted ? ($d['fsSize'] ?? 0) : ($d['size'] ?? 0)));
|
||
$used_kb = (int)($isParity ? 0 : ($d['fsUsed'] ?? 0));
|
||
if ($size_kb <= 0) return null;
|
||
$tempRaw = trim($d['temp'] ?? '');
|
||
return [
|
||
'name' => $name,
|
||
'device' => $d['device'] ?? $key,
|
||
'role' => $role,
|
||
// disks.ini is KiB. This path was arithmetically right and still disagreed with the API
|
||
// path by 2.4% and with Unraid's own Main page by 7.4% — it produced GiB under a "GB"
|
||
// name. Decimal GB, so both sources of the same disk now land on the same number.
|
||
'size_gb' => round($size_kb * 1024 / 1e9, 1),
|
||
'used_gb' => round($used_kb * 1024 / 1e9, 1),
|
||
'pct' => (!$isParity && $size_kb > 0) ? round($used_kb / $size_kb * 100, 1) : null,
|
||
'temp' => is_numeric($tempRaw) ? (int)$tempRaw : null,
|
||
'transport' => $d['transport'] ?? 'ata',
|
||
'mounted' => $mounted,
|
||
'status' => $d['status'] ?? '',
|
||
];
|
||
}
|
||
|
||
function vv_ups_stats(): array {
|
||
$raw = shell_exec('apcaccess 2>/dev/null') ?: '';
|
||
if (!$raw) return ['available' => false];
|
||
|
||
$fields = [];
|
||
foreach (explode("\n", $raw) as $line) {
|
||
if (preg_match('/^(\w+)\s*:\s*(.+)$/', trim($line), $m)) {
|
||
$fields[trim($m[1])] = trim($m[2]);
|
||
}
|
||
}
|
||
if (empty($fields)) return ['available' => false];
|
||
|
||
$parse_num = fn(string $k) => isset($fields[$k]) ? (float)$fields[$k] : null;
|
||
|
||
$loadPct = $parse_num('LOADPCT');
|
||
$nomPower = $parse_num('NOMPOWER');
|
||
$watts = ($loadPct !== null && $nomPower !== null) ? round($loadPct / 100 * $nomPower) : null;
|
||
|
||
return [
|
||
'available' => true,
|
||
'model' => $fields['MODEL'] ?? '',
|
||
'status' => trim(explode(' ', $fields['STATUS'] ?? 'UNKNOWN')[0]),
|
||
'line_v' => $parse_num('LINEV'),
|
||
'output_v' => $parse_num('OUTPUTV'),
|
||
'load_pct' => $loadPct,
|
||
'nom_power' => $nomPower,
|
||
'watts' => $watts,
|
||
'bcharge' => $parse_num('BCHARGE'),
|
||
'timeleft' => $parse_num('TIMELEFT'),
|
||
'num_xfers' => (int)($fields['NUMXFERS'] ?? 0),
|
||
'on_batt_s' => $parse_num('CUMONBATT'),
|
||
'selftest' => $fields['SELFTEST'] ?? '',
|
||
];
|
||
}
|
||
|
||
// CPU temperature in °C, or null if no sensor answers.
|
||
//
|
||
// Mirror of platform_get_cpu_temp() in Plugin/unraid/adapter.sh — the two must agree, because the
|
||
// Monitor card and the stability watchdog display and act on the same number, and for months they
|
||
// did not. The PHP side scraped every decimal off the sensor line and took the largest, which on
|
||
// any board printing "(high = +80.0 C, crit = +100.0 C)" is the critical threshold: HOST2 reported
|
||
// a flat 100°C while sitting at 63. The shell side took the line's last field, which on that same
|
||
// board is the literal ")", so its check silently never fired.
|
||
//
|
||
// Both defects came from parsing a line that carries three temperatures when only one of them is a
|
||
// reading. The parenthetical is stripped before any digit is read.
|
||
//
|
||
// Tdie before Tctl for the AMD reason: Tctl carries a fixed offset (+27°C on Threadripper) and is
|
||
// a control value, not a measurement — it is why HOST1 read 70 while the die was at 43.
|
||
function vv_cpu_temp(): ?int {
|
||
$out = shell_exec('sensors 2>/dev/null') ?: '';
|
||
if (trim($out) === '') return null;
|
||
|
||
$out = preg_replace('/\(.*$/m', '', $out); // drop "(high = ..., crit = ...)"
|
||
|
||
foreach (['Tdie', 'Package id 0', 'CPU Temp', 'Core '] as $label) {
|
||
$best = null;
|
||
foreach (explode("\n", $out) as $line) {
|
||
if (stripos(ltrim($line), $label) !== 0) continue;
|
||
if (!preg_match('/([+-]?\d+\.\d+)/', $line, $m)) continue;
|
||
$v = (float)$m[1];
|
||
// Max within a label: a multi-die part publishes one line per die, and the hottest is
|
||
// the one worth acting on.
|
||
if ($best === null || $v > $best) $best = $v;
|
||
}
|
||
if ($best !== null) return (int)round($best);
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function vv_parity_status(): array {
|
||
$var = [];
|
||
foreach (@file('/var/local/emhttp/var.ini') ?: [] as $line) {
|
||
if (preg_match('/^(\w+)="([^"]*)"/', $line, $m)) $var[$m[1]] = $m[2];
|
||
}
|
||
|
||
$numDisabled = (int)($var['mdNumDisabled'] ?? 0);
|
||
$numMissing = (int)($var['mdNumMissing'] ?? 0);
|
||
$exitCode = (int)($var['sbSyncExit'] ?? 0);
|
||
$errors = (int)($var['sbSyncErrs'] ?? 0);
|
||
|
||
// How many disks are actually being emulated, which is not mdNumDisabled.
|
||
//
|
||
// mdNumDisabled counts every slot the array marks disabled, including a parity slot that was
|
||
// never populated. HOST2 reports mdNumDisabled=1 with mdNumMissing=0, and the slot is parity2
|
||
// at DISK_NP_DSBL — an empty second-parity slot. Unraid's own Main page shows nothing for it,
|
||
// while this card said "Emulating 1 disk" and turned the banner amber.
|
||
//
|
||
// Parity is never emulated: reconstructing a parity disk from parity is not a thing. Only data
|
||
// slots are, so the count is taken from disks.ini and restricted to them. A data disk that is
|
||
// disabled *is* emulated whether or not it is physically present — a pulled failed drive reads
|
||
// DISK_NP_DSBL and is genuinely being served from parity — so presence is not the test here,
|
||
// role is.
|
||
//
|
||
// A disabled parity slot is still worth knowing about when a real parity disk fails, so it is
|
||
// counted separately rather than discarded.
|
||
//
|
||
// Both counts require the slot to have an assigned identity. A slot that was never populated
|
||
// reads id="" device="" size="0" — parity2 on HOST2 is exactly that — while a disk that failed
|
||
// and was pulled keeps its id, because the array remembers the assignment it is emulating.
|
||
// Presence is therefore the wrong test and identity is the right one: it separates "no disk was
|
||
// ever here" from "the disk that belongs here is gone", which look identical in the status
|
||
// field alone and mean opposite things.
|
||
$numEmulated = 0;
|
||
$numParityDisabled = 0;
|
||
$slot = ''; $slotId = ''; $slotStatus = '';
|
||
$tally = function () use (&$slot, &$slotId, &$slotStatus, &$numEmulated, &$numParityDisabled) {
|
||
if ($slot === '' || $slotId === '') return;
|
||
if (strpos($slotStatus, 'DSBL') === false) return;
|
||
if (preg_match('/^disk\d+$/', $slot)) $numEmulated++;
|
||
elseif (preg_match('/^parity\d*$/', $slot)) $numParityDisabled++;
|
||
};
|
||
foreach (@file('/var/local/emhttp/disks.ini') ?: [] as $line) {
|
||
if (preg_match('/^\["([^"]+)"\]/', $line, $m)) {
|
||
$tally(); // close the previous slot
|
||
$slot = $m[1]; $slotId = ''; $slotStatus = '';
|
||
continue;
|
||
}
|
||
if (preg_match('/^id="([^"]*)"/', $line, $m)) $slotId = trim($m[1]);
|
||
if (preg_match('/^status="([^"]*)"/', $line, $m)) $slotStatus = $m[1];
|
||
}
|
||
$tally(); // and the last one, which has no header after it
|
||
// Emulated (DISK_DSBL) disks are protected by parity and don't make parity invalid.
|
||
// True invalidity: sync errors on the last check, or unprotectable missing slots.
|
||
$isValid = $errors === 0 && $numMissing === 0;
|
||
$inProgress = ($var['mdResync'] ?? '0') !== '0';
|
||
$resyncAction = trim($var['mdResyncAction'] ?? '');
|
||
$resyncPos = (int)($var['mdResyncPos'] ?? 0);
|
||
$resyncSize = (int)($var['mdResyncSize'] ?? 1);
|
||
$resyncPct = $resyncSize > 0 ? round($resyncPos / $resyncSize * 100, 1) : 0;
|
||
|
||
// Last check from log
|
||
$lastDate = null; $lastDuration = 0; $lastSpeed = 0; $lastErrors = 0; $lastExit = 0;
|
||
$logFile = '/boot/config/parity-checks.log';
|
||
if (file_exists($logFile)) {
|
||
$lines = file($logFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ?: [];
|
||
if ($lines) {
|
||
$p = explode('|', trim(end($lines)));
|
||
$lastDate = trim($p[0] ?? '');
|
||
$lastDuration = (int)($p[1] ?? 0);
|
||
$lastSpeed = (int)($p[2] ?? 0);
|
||
$lastExit = (int)($p[3] ?? 0);
|
||
$lastErrors = (int)($p[4] ?? 0);
|
||
}
|
||
}
|
||
|
||
// Parse last date string to timestamp
|
||
$lastTs = $lastDate ? strtotime($lastDate) : null;
|
||
|
||
// Next scheduled check from cron
|
||
$nextTs = null;
|
||
$cronFile = '/boot/config/plugins/dynamix/parity-check.cron';
|
||
foreach (@file($cronFile) ?: [] as $line) {
|
||
$line = trim($line);
|
||
if ($line === '' || $line[0] === '#') continue;
|
||
if (!str_contains($line, 'mdcmd')) continue;
|
||
$p = preg_split('/\s+/', $line);
|
||
// cron: min hour dom month dow command...
|
||
if (count($p) >= 5 && is_numeric($p[0]) && is_numeric($p[1]) && is_numeric($p[2])) {
|
||
$next = new DateTime('now');
|
||
$next->setTime((int)$p[1], (int)$p[0], 0);
|
||
$next->setDate((int)$next->format('Y'), (int)$next->format('n'), (int)$p[2]);
|
||
if ($next->getTimestamp() <= time()) $next->modify('+1 month');
|
||
$nextTs = $next->getTimestamp();
|
||
}
|
||
break;
|
||
}
|
||
|
||
$exitMap = ['0' => 'Completed', '-4' => 'Aborted', '-5' => 'Cancelled'];
|
||
return [
|
||
'valid' => $isValid,
|
||
// mdNumDisabled as the array reports it, kept because it is Unraid's own number and
|
||
// anything comparing against the WebGUI wants it. Not what the card labels, though —
|
||
// num_emulated is the one that means "data is being served from parity".
|
||
'num_disabled' => $numDisabled,
|
||
'num_emulated' => $numEmulated,
|
||
'num_parity_disabled' => $numParityDisabled,
|
||
'num_missing' => $numMissing,
|
||
'in_progress' => $inProgress,
|
||
'resync_action' => $resyncAction,
|
||
'resync_pct' => $resyncPct,
|
||
'exit_code' => $exitCode,
|
||
'exit_label' => $exitMap[(string)$lastExit] ?? 'Unknown',
|
||
'errors' => $lastErrors,
|
||
'last_date' => $lastDate,
|
||
'last_ts' => $lastTs,
|
||
'last_duration' => $lastDuration,
|
||
'last_speed_mb' => $lastSpeed > 0 ? round($lastSpeed / 1048576, 1) : null,
|
||
'next_ts' => $nextTs,
|
||
];
|
||
}
|
||
|
||
// Returns a map of disk name → fsUsed in KB from disks.ini.
|
||
// Unraid keeps this updated even after spindown, so it's the authoritative source
|
||
// for used space when the API reports 0 because the filesystem is unmounted.
|
||
function _vv_ini_used_kb(): array {
|
||
$ini = @parse_ini_file('/var/local/emhttp/disks.ini', true) ?: [];
|
||
$out = [];
|
||
foreach ($ini as $key => $d) {
|
||
$name = $d['name'] ?? $key;
|
||
$out[$name] = (int)($d['fsUsed'] ?? 0);
|
||
}
|
||
return $out;
|
||
}
|
||
|
||
function vv_storage_pools(): array {
|
||
// ── API path ──────────────────────────────────────────────────────────────
|
||
$api = vv_api_data();
|
||
if ($api && isset($api['array']['caches'])) {
|
||
$ini_used = _vv_ini_used_kb();
|
||
$out = [];
|
||
foreach ($api['array']['caches'] as $d) {
|
||
$entry = vv_api_disk_entry($d, 'data', $ini_used[$d['name'] ?? ''] ?? 0);
|
||
if ($entry) $out[] = $entry;
|
||
}
|
||
if (!empty($out)) {
|
||
usort($out, fn($a, $b) => strcmp($a['name'], $b['name']));
|
||
return $out;
|
||
}
|
||
}
|
||
|
||
// ── Local fallback ────────────────────────────────────────────────────────
|
||
vv_api_record_fallback('storage_pools');
|
||
$ini = @parse_ini_file('/var/local/emhttp/disks.ini', true) ?: [];
|
||
$out = [];
|
||
foreach ($ini as $key => $d) {
|
||
if (($d['type'] ?? '') !== 'Cache') continue;
|
||
if (($d['fsStatus'] ?? '') !== 'Mounted') continue;
|
||
$entry = vv_disk_entry($d, $key);
|
||
if ($entry) $out[] = $entry;
|
||
}
|
||
usort($out, fn($a, $b) => strcmp($a['name'], $b['name']));
|
||
return $out;
|
||
}
|
||
|
||
function vv_array_disks(): array {
|
||
// ── API path ──────────────────────────────────────────────────────────────
|
||
$api = vv_api_data();
|
||
if ($api && (isset($api['array']['parities']) || isset($api['array']['disks']))) {
|
||
$ini_used = _vv_ini_used_kb();
|
||
$parity = [];
|
||
$data = [];
|
||
foreach ($api['array']['parities'] ?? [] as $d) {
|
||
$entry = vv_api_disk_entry($d, 'parity');
|
||
if ($entry) $parity[] = $entry;
|
||
}
|
||
foreach ($api['array']['disks'] ?? [] as $d) {
|
||
$entry = vv_api_disk_entry($d, 'data', $ini_used[$d['name'] ?? ''] ?? 0);
|
||
if ($entry) $data[] = $entry;
|
||
}
|
||
if (!empty($parity) || !empty($data)) {
|
||
usort($parity, fn($a, $b) => strnatcmp($a['name'], $b['name']));
|
||
usort($data, fn($a, $b) => strnatcmp($a['name'], $b['name']));
|
||
return array_merge($parity, $data);
|
||
}
|
||
}
|
||
|
||
// ── Local fallback ────────────────────────────────────────────────────────
|
||
vv_api_record_fallback('array_disks');
|
||
$ini = @parse_ini_file('/var/local/emhttp/disks.ini', true) ?: [];
|
||
$parity = [];
|
||
$data = [];
|
||
foreach ($ini as $key => $d) {
|
||
$type = $d['type'] ?? '';
|
||
if ($type === 'Parity') {
|
||
$entry = vv_disk_entry($d, $key, 'parity');
|
||
if ($entry) $parity[] = $entry;
|
||
} elseif ($type === 'Data') {
|
||
$entry = vv_disk_entry($d, $key, 'data');
|
||
if ($entry) $data[] = $entry;
|
||
}
|
||
}
|
||
usort($parity, fn($a, $b) => strnatcmp($a['name'], $b['name']));
|
||
usort($data, fn($a, $b) => strnatcmp($a['name'], $b['name']));
|
||
return array_merge($parity, $data);
|
||
}
|
||
|
||
function vv_disk_io_rates(): array {
|
||
$snapFile = VV_CACHE_DIR . '/vv_diskio_snap.json';
|
||
$now = microtime(true);
|
||
|
||
// Read current whole-disk stats from /proc/diskstats
|
||
$current = [];
|
||
foreach (@file('/proc/diskstats', FILE_IGNORE_NEW_LINES) ?: [] as $line) {
|
||
$p = preg_split('/\s+/', trim($line));
|
||
if (count($p) < 14) continue;
|
||
$dev = $p[2];
|
||
// Keep only whole disks: sda/sdb, nvme0n1, md*, not sda1/nvme0n1p1
|
||
if (!preg_match('/^(sd[a-z]+|nvme\d+n\d+|md\d+)$/', $dev)) continue;
|
||
$current[$dev] = [(int)$p[5], (int)$p[9]]; // [sectors_read, sectors_written]
|
||
}
|
||
|
||
// Load previous snapshot
|
||
$snap = @json_decode(@file_get_contents($snapFile) ?: '', true) ?: [];
|
||
$prevTime = (float)($snap['t'] ?? $now);
|
||
$prev = $snap['d'] ?? [];
|
||
|
||
// Save current snapshot
|
||
@file_put_contents($snapFile, json_encode(['t' => $now, 'd' => $current], JSON_UNESCAPED_UNICODE));
|
||
|
||
$dt = max(0.5, $now - $prevTime);
|
||
$out = [];
|
||
foreach ($current as $dev => [$rs, $ws]) {
|
||
$entry = [
|
||
'tr' => round($rs * 512 / 1073741824, 2), // cumulative GB read
|
||
'tw' => round($ws * 512 / 1073741824, 2), // cumulative GB written
|
||
];
|
||
if (isset($prev[$dev])) {
|
||
[$prs, $pws] = $prev[$dev];
|
||
$r = max(0.0, ($rs - $prs) * 512 / $dt / 1048576);
|
||
$w = max(0.0, ($ws - $pws) * 512 / $dt / 1048576);
|
||
if ($r > 0.01) $entry['r'] = round($r, 1);
|
||
if ($w > 0.01) $entry['w'] = round($w, 1);
|
||
}
|
||
$out[$dev] = $entry;
|
||
}
|
||
return $out;
|
||
}
|
||
|
||
function vv_disk_thresholds(): array {
|
||
$cfg = @file_get_contents('/boot/config/plugins/dynamix/dynamix.cfg') ?: '';
|
||
$get = function(string $key) use ($cfg): ?int {
|
||
return preg_match('/^\s*' . preg_quote($key, '/') . '\s*=\s*"?(\d+)"?/m', $cfg, $m)
|
||
? (int)$m[1] : null;
|
||
};
|
||
return [
|
||
'util_warn' => $get('warning') ?? 70,
|
||
'util_crit' => $get('critical') ?? 90,
|
||
'hdd_warn' => $get('hot') ?? 45,
|
||
'hdd_crit' => $get('max') ?? 55,
|
||
'ssd_warn' => $get('hotssd') ?? 60,
|
||
'ssd_crit' => $get('maxssd') ?? 70,
|
||
];
|
||
}
|
||
|
||
// Fetch a lightweight snapshot from each remote host that has an API key configured.
|
||
// Results are cached in /tmp for 30 seconds so rapid monitor polls don't hammer remote hosts.
|
||
function vv_remote_hosts_stats(): array {
|
||
// Read ALL conf files — remote host keys live in their own host*.conf, not the current host's.
|
||
//
|
||
// The RAM cache is read first and wins, because it is the only current copy of a partner conf.
|
||
// conf_sync.sh pulls each partner's live conf into VV_CONF_RAM_CACHE_DIR; load_config.sh has
|
||
// sourced partner vars from there rather than from disk for as long as the cache has existed,
|
||
// but this glob never did — so PHP and bash could disagree about the same partner. Any
|
||
// host*.conf still sitting in CONF_DIR for a host that is not this one is a leftover from
|
||
// before sparse checkout, and is stale by definition.
|
||
$vars = vv_conf_vars();
|
||
$confFiles = array_merge(
|
||
glob(VV_CONF_RAM_CACHE_DIR . '/host*.conf') ?: [],
|
||
glob(CONF_DIR . '/host*.conf') ?: []
|
||
);
|
||
foreach ($confFiles as $f) {
|
||
$raw = file_get_contents($f) ?: '';
|
||
preg_match_all('/^\s*([A-Z0-9_]+)\s*=\s*["\']?([^"\'#\n]*?)["\']?\s*(?:#.*)?$/m', $raw, $m);
|
||
foreach ($m[1] as $i => $key) {
|
||
if (!isset($vars[$key])) $vars[$key] = trim($m[2][$i]);
|
||
}
|
||
}
|
||
$myHost = vv_detect_host();
|
||
$hostIds = array_filter(array_keys($vars), fn($k) => preg_match('/^HOST\d+$/', $k) && ($vars[$k] ?? '') !== '');
|
||
sort($hostIds);
|
||
|
||
$results = [];
|
||
foreach ($hostIds as $id) {
|
||
if (strtolower($id) === strtolower($myHost)) continue;
|
||
// Background cache written by remote_arr_cache_writer.sh every 2h — use it if present.
|
||
$bgCache = VV_CACHE_DIR . '/monitor_remote_' . strtolower($id) . '.json';
|
||
if (file_exists($bgCache)) {
|
||
$cached = json_decode(file_get_contents($bgCache), true);
|
||
if ($cached) {
|
||
$cached['cache_age'] = time() - (int)filemtime($bgCache);
|
||
$results[$id] = $cached;
|
||
continue;
|
||
}
|
||
}
|
||
|
||
// No background cache yet — fall back to live call (uses 30s inline cache).
|
||
$key = $vars[strtoupper($id) . '_UNRAID_API_KEY'] ?? '';
|
||
if (!$key) {
|
||
$results[$id] = ['available' => false, 'no_api_key' => true,
|
||
'host_id' => $id, 'hostname' => $vars[$id]];
|
||
continue;
|
||
}
|
||
|
||
$cacheFile = VV_CACHE_DIR . "/vv_remote_{$id}.json";
|
||
if (file_exists($cacheFile) && (time() - filemtime($cacheFile)) < 30) {
|
||
$cached = json_decode(file_get_contents($cacheFile), true);
|
||
if ($cached) { $results[$id] = $cached; continue; }
|
||
}
|
||
|
||
$gql = '{
|
||
info { os { hostname uptime release } cpu { brand threads cores } }
|
||
metrics { cpu { percentTotal } memory { percentTotal total used available } }
|
||
array {
|
||
state
|
||
disks { fsSize fsUsed temp }
|
||
caches { fsSize fsUsed temp }
|
||
parities { temp }
|
||
}
|
||
vms { domains { name } }
|
||
}';
|
||
$data = vv_unraid_api_query(strtolower($id), $gql, 4, $key);
|
||
|
||
if (!$data) {
|
||
$entry = ['available' => false, 'host_id' => $id, 'hostname' => $vars[$id]];
|
||
file_put_contents($cacheFile, json_encode($entry));
|
||
$results[$id] = $entry;
|
||
continue;
|
||
}
|
||
|
||
$os = $data['info']['os'] ?? [];
|
||
$cpu = $data['info']['cpu'] ?? [];
|
||
$mMem = $data['metrics']['memory'] ?? [];
|
||
|
||
$memPct = round((float)($mMem['percentTotal'] ?? 0));
|
||
if ($memPct === 0) {
|
||
$totalBytes = (float)($mMem['total'] ?? 0);
|
||
$availBytes = (float)($mMem['available'] ?? 0);
|
||
$memPct = $totalBytes > 0 ? (int)round(($totalBytes - $availBytes) / $totalBytes * 100) : 0;
|
||
}
|
||
$memTotalGb = isset($mMem['total']) ? _vv_api_bytes_to_gib((float)$mMem['total']) : 0;
|
||
|
||
$uptimeRaw = $os['uptime'] ?? '';
|
||
if (is_numeric($uptimeRaw)) {
|
||
$uptimeSec = (int)$uptimeRaw;
|
||
$days = intdiv($uptimeSec, 86400);
|
||
$hours = intdiv($uptimeSec % 86400, 3600);
|
||
$mins = intdiv($uptimeSec % 3600, 60);
|
||
$uptime = ($days ? "{$days}d " : '') . ($hours ? "{$hours}h " : '') . "{$mins}m";
|
||
} else {
|
||
$uptimeSec = 0;
|
||
$uptime = $uptimeRaw ?: '—';
|
||
}
|
||
|
||
$nodeMetrics = vv_api_node_metrics($data);
|
||
$entry = array_merge([
|
||
'available' => true,
|
||
'host_id' => $id,
|
||
'hostname' => $os['hostname'] ?? $vars[$id],
|
||
'version' => $os['release'] ?? '',
|
||
'uptime' => $uptime,
|
||
'uptime_sec' => $uptimeSec,
|
||
'cpu_load' => $nodeMetrics['cpu_pct'] ?? 0,
|
||
'cpu_threads' => (int)($cpu['threads'] ?? 0),
|
||
'mem_total_gb' => $memTotalGb,
|
||
'mem_used_pct' => $memPct,
|
||
'array_state' => $data['array']['state'] ?? 'UNKNOWN',
|
||
], $nodeMetrics);
|
||
file_put_contents($cacheFile, json_encode($entry));
|
||
$results[$id] = $entry;
|
||
}
|
||
return $results;
|
||
}
|
||
|
||
function vv_log_tail(string $path, int $lines): string {
|
||
$fp = @fopen($path, 'r');
|
||
if (!$fp) return '';
|
||
fseek($fp, 0, SEEK_END);
|
||
$size = ftell($fp);
|
||
if ($size <= 0) { fclose($fp); return ''; }
|
||
$chunk = min($size, 4096);
|
||
fseek($fp, -$chunk, SEEK_END);
|
||
$data = fread($fp, $chunk);
|
||
fclose($fp);
|
||
$all = explode("\n", $data ?: '');
|
||
return implode("\n", array_slice($all, -$lines));
|
||
}
|
||
|
||
// See VV_CONF_ARRAY_BODY in config.php for why this cannot stop at the first `)`.
|
||
function vv_parse_bash_array(string $raw, string $varName): array {
|
||
if (!preg_match('/^\s*' . preg_quote($varName, '/') . '\s*=\s*\(' . VV_CONF_ARRAY_BODY . '/ms',
|
||
$raw, $m)) return [];
|
||
// Group 2 exists only when the multi-line branch matched; group 1 is the single-line body.
|
||
$body = isset($m[2]) ? $m[2] : ($m[1] ?? '');
|
||
$items = [];
|
||
foreach (explode("\n", $body) as $line) {
|
||
$line = trim(preg_replace('/#.*$/', '', $line), " \t\"'");
|
||
if ($line !== '') $items[] = $line;
|
||
}
|
||
return $items;
|
||
}
|
||
|
||
function vv_transcode_sessions(): array {
|
||
$v = vv_conf_vars();
|
||
$stateDir = rtrim($v['STATE_DIR'] ?? STATE_DIR, '/');
|
||
$stateFile = "$stateDir/transcode_state.db";
|
||
if (!file_exists($stateFile)) return ['available' => false];
|
||
|
||
$raw = [];
|
||
foreach (file($stateFile) ?: [] as $line) {
|
||
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
||
$raw[trim($k)] = trim($v);
|
||
}
|
||
|
||
$target = $raw['current_target'] ?? '';
|
||
$lastFlip = (int)($raw['last_flip_time'] ?? 0);
|
||
$flipCount = (int)($raw['flip_count_hour'] ?? 0);
|
||
$isRamdisk = str_contains($target, 'ramdisk');
|
||
|
||
// Count active sessions: subdirs (legacy transcode) + unique hex prefixes (Live TV / Direct Stream HLS)
|
||
$ramdiskPath = '/mnt/ramdisk_transcodes/transcoding-temp';
|
||
$ramSessions = count(glob("$ramdiskPath/*/", GLOB_ONLYDIR) ?: []);
|
||
$maxAge = (int)(vv_conf_vars()['TRANSCODE_MAX_AGE'] ?? 20);
|
||
$activeFiles = 0;
|
||
$cutoff = time() - $maxAge * 60;
|
||
$flatPrefixes = [];
|
||
if (is_dir($ramdiskPath)) {
|
||
foreach (new DirectoryIterator($ramdiskPath) as $f) {
|
||
if (!$f->isFile()) continue;
|
||
if (preg_match('/^([0-9a-f]{16,})/', $f->getFilename(), $m)) {
|
||
$flatPrefixes[$m[1]] = true;
|
||
}
|
||
if ($f->getMTime() >= $cutoff) $activeFiles++;
|
||
}
|
||
}
|
||
$ramSessions += count($flatPrefixes);
|
||
|
||
// SSD path: first transcoding-temp mount that is not a RAM filesystem (tmpfs/ramfs)
|
||
$ssdPath = '';
|
||
$ssdSessions = 0;
|
||
foreach (glob('/mnt/*/transcoding-temp/', GLOB_ONLYDIR) ?: [] as $p) {
|
||
$parts = explode('/', rtrim($p, '/'));
|
||
array_pop($parts);
|
||
$mount = implode('/', $parts) ?: '/';
|
||
$fsType = trim(shell_exec('findmnt -n -o FSTYPE ' . escapeshellarg($mount) . ' 2>/dev/null') ?: '');
|
||
if ($fsType === 'tmpfs' || $fsType === 'ramfs') continue;
|
||
$ssdPath = $p;
|
||
break;
|
||
}
|
||
$ssd = ['available' => false];
|
||
if ($ssdPath) {
|
||
$ssdSessions = count(glob($ssdPath . '/*/', GLOB_ONLYDIR) ?: []);
|
||
$parts = explode('/', rtrim($ssdPath, '/'));
|
||
array_pop($parts);
|
||
$ssdMount = implode('/', $parts) ?: '/';
|
||
$ssd = vv_df($ssdMount);
|
||
}
|
||
|
||
// Ramdisk disk usage
|
||
$rd = vv_df('/mnt/ramdisk_transcodes');
|
||
|
||
// Last cleanup values from transcode management log
|
||
$lastRdFreed = null;
|
||
$lastSsdFreed = null;
|
||
$logFile = LOG_DIR . '/Orchestrators/transcode_management.log';
|
||
if (file_exists($logFile)) {
|
||
$lines = file($logFile, FILE_IGNORE_NEW_LINES) ?: [];
|
||
foreach (array_reverse($lines) as $line) {
|
||
if ($lastRdFreed === null && preg_match('/Ramdisk freed:\s*(\S+)/u', $line, $m))
|
||
$lastRdFreed = $m[1];
|
||
if ($lastSsdFreed === null && preg_match('/SSD freed:\s*(\S+)/u', $line, $m))
|
||
$lastSsdFreed = $m[1];
|
||
if ($lastRdFreed !== null && $lastSsdFreed !== null) break;
|
||
}
|
||
}
|
||
|
||
return [
|
||
'available' => true,
|
||
'current_target' => $target,
|
||
'is_ramdisk' => $isRamdisk,
|
||
'flip_count_hour' => $flipCount,
|
||
'last_flip_time' => $lastFlip,
|
||
'last_flip_ago' => $lastFlip > 0 ? time() - $lastFlip : null,
|
||
'ram_sessions' => $ramSessions,
|
||
'ssd_sessions' => $ssdSessions,
|
||
'active_files' => $activeFiles,
|
||
'ramdisk' => $rd,
|
||
'ssd' => $ssd,
|
||
'last_rd_freed' => $lastRdFreed,
|
||
'last_ssd_freed' => $lastSsdFreed,
|
||
];
|
||
}
|
||
|
||
// ── Notification ─────────────────────────────────────────────────────────────────────────────
|
||
// Hands a message to common.sh's notify(), rather than reimplementing it here.
|
||
//
|
||
// notify() is the one place that knows this installation's channels — Unraid's native notifier
|
||
// via the adapter, and the per-host Discord webhook — and which of them are switched on. A PHP
|
||
// copy would be a second answer to "how does this machine reach its operator", and the two would
|
||
// drift the first time a channel is added.
|
||
//
|
||
// detect_hosts() is called explicitly and that is not optional. load_config.sh deliberately does
|
||
// not call it — its header says so — and MY_DISCORD_WEBHOOK is set by detect_hosts() from
|
||
// HOST<n>_DISCORD_WEBHOOK. Skipping it yields a notification that reaches the Unraid GUI and
|
||
// silently never reaches Discord, which is the failure that looks like success.
|
||
//
|
||
// Single-line messages only. notify() builds its Discord payload with printf into a JSON string
|
||
// literal, so a raw newline in the message produces invalid JSON and the webhook rejects it.
|
||
// Callers separate with ' · '.
|
||
// Is there anywhere for a notification to go?
|
||
//
|
||
// notify() exits 0 whether or not it did anything — with NOTIFY_UNRAID false and no webhook it
|
||
// logs a line and returns success, because from its point of view nothing went wrong. A caller
|
||
// that treats that as delivered will mark its work as told and go quiet about it forever, so the
|
||
// question "is any channel switched on" is answered here instead, from the same two settings
|
||
// notify() itself consults.
|
||
function vv_notify_available(): bool {
|
||
$conf = vv_conf_vars();
|
||
if (strtolower(trim((string)($conf['NOTIFY_UNRAID'] ?? 'false'))) === 'true') return true;
|
||
$hook = strtoupper(vv_detect_host()) . '_DISCORD_WEBHOOK';
|
||
return trim((string)($conf[$hook] ?? '')) !== '';
|
||
}
|
||
|
||
function vv_notify(string $message, string $subject, string $severity = 'normal'): bool {
|
||
$loader = SCRIPTS_DIR . '/load_config.sh';
|
||
if (!is_file($loader)) return false;
|
||
if (!vv_notify_available()) return false;
|
||
if (!in_array($severity, ['normal', 'warning', 'alert'], true)) $severity = 'normal';
|
||
|
||
// Newlines are stripped rather than refused: a caller that accidentally includes one should
|
||
// still reach the operator, just on one line.
|
||
$message = trim(preg_replace('/\s*[\r\n]+\s*/', ' | ', $message));
|
||
if ($message === '') return false;
|
||
|
||
// Typographic characters do not survive Unraid's notifier — it dropped an em dash outright
|
||
// and left the double space behind it, which was found by sending one and reading what
|
||
// arrived. Every string in this codebase is full of them, so they are folded to ASCII here
|
||
// rather than asked of each caller. Anything else non-ASCII is left alone: a share name with
|
||
// an accent should arrive imperfectly rather than not at all.
|
||
$message = strtr($message, ['—' => '-', '–' => '-', '·' => '|', '→' => '->',
|
||
'“' => '"', '”' => '"', '‘' => "'", '’' => "'", '…' => '...']);
|
||
$subject = strtr($subject, ['—' => '-', '–' => '-', '·' => '|', '→' => '->',
|
||
'“' => '"', '”' => '"', '‘' => "'", '’' => "'", '…' => '...']);
|
||
|
||
$script = 'source ' . escapeshellarg($loader) . ' >/dev/null 2>&1 || exit 91; '
|
||
. 'detect_hosts >/dev/null 2>&1; '
|
||
. 'notify ' . escapeshellarg($message) . ' ' . escapeshellarg($subject) . ' '
|
||
. escapeshellarg($severity) . ' >/dev/null 2>&1';
|
||
|
||
$out = []; $rc = 0;
|
||
exec('bash -c ' . escapeshellarg($script), $out, $rc);
|
||
return $rc === 0;
|
||
}
|