diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 31ca137..7debb44 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -493,6 +493,7 @@ # make five strikes five weeks. One NPM list call; it is what gives the Certs tab its # per-domain renewal and failure counts. "Plugin/unraid/Tools/cert_history.sh" # record cert renewals, failures and age per domain + "Plugin/unraid/Tools/npm_access_stats.sh" # aggregate NPM per-host access logs into request and byte totals ) # Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart. diff --git a/Plugin/unraid/Tools/npm_access_stats.php b/Plugin/unraid/Tools/npm_access_stats.php new file mode 100644 index 0000000..00ae377 --- /dev/null +++ b/Plugin/unraid/Tools/npm_access_stats.php @@ -0,0 +1,225 @@ +/dev/null'); + foreach (explode("\n", trim((string) $out)) as $line) { + // println space-separates its arguments, so the mapping arrives as "src : dst". + $line = str_replace(' : ', ':', trim($line)); + [$src, $dst] = array_pad(explode(':', $line, 2), 2, ''); + if ($dst === '/config' && $src !== '' && is_dir("$src/log")) return "$src/log"; + if ($dst === '/data' && $src !== '' && is_dir("$src/logs")) return "$src/logs"; + } + return ''; +} + +function vv_npm_stats_read(): array { + $p = vv_npm_stats_path(); + if (!is_file($p)) return ['hosts' => []]; + $j = json_decode((string) @file_get_contents($p), true); + if (!is_array($j) || !isset($j['hosts']) || !is_array($j['hosts'])) return []; + return $j; +} + +function vv_npm_stats_write(array $data): bool { + $p = vv_npm_stats_path(); + if (!is_dir(dirname($p)) && !@mkdir(dirname($p), 0755, true)) return false; + $data['updated'] = time(); + $json = json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + if ($json === false) return false; + $tmp = $p . '.vv.tmp'; + if (@file_put_contents($tmp, $json) === false) return false; + if (json_decode((string) @file_get_contents($tmp), true) === null) { @unlink($tmp); return false; } + if (!@rename($tmp, $p)) { @unlink($tmp); return false; } + return true; +} + +// ── Status ──────────────────────────────────────────────────────────────────── +if ($status) { + $s = vv_npm_stats_read(); + if (!$s) { echo "npm_access.json is unreadable\n"; exit(0); } + printf("%-6s %12s %12s %8s %8s %8s %s\n", 'host', 'requests', 'sent', '2xx', '4xx', '5xx', 'last hit'); + foreach ($s['hosts'] ?? [] as $id => $h) { + printf("%-6s %12s %12s %8s %8s %8s %s\n", $id, + number_format($h['requests'] ?? 0), vv_npm_bytes($h['bytes'] ?? 0), + number_format($h['s2xx'] ?? 0), number_format($h['s4xx'] ?? 0), + number_format($h['s5xx'] ?? 0), + !empty($h['last_seen']) ? date('Y-m-d H:i', $h['last_seen']) : '-'); + } + exit(0); +} + +// Decimal, because this is a quantity of traffic and every network tool that will be compared +// against it is decimal too. Memory is the binary one. +function vv_npm_bytes(float $b): string { + $u = ['B', 'kB', 'MB', 'GB', 'TB']; + $i = 0; + while ($b >= 1000 && $i < count($u) - 1) { $b /= 1000; $i++; } + return round($b, $b < 10 && $i ? 1 : 0) . ' ' . $u[$i]; +} + +$lock = @fopen(sys_get_temp_dir() . '/vv_npm_access.lock', 'c'); +if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) { echo "another pass is running\n"; exit(0); } + +try { + $dir = vv_npm_log_dir(); + if ($dir === '') { echo "NPM log directory not found — set NPM_LOG_DIR\n"; exit(0); } + + $store = $reset ? ['hosts' => []] : vv_npm_stats_read(); + if (!$store) { echo "npm_access.json is malformed — refusing to overwrite it\n"; exit(1); } + $hosts = $store['hosts'] ?? []; + + $files = glob("$dir/proxy-host-*_access.log") ?: []; + if (!$files) { echo "no per-host access logs in $dir\n"; exit(0); } + + $t0 = microtime(true); + $readTotal = 0; $newLines = 0; + + foreach ($files as $f) { + if (!preg_match('/proxy-host-(\d+)_access\.log$/', $f, $m)) continue; + $id = (string) (int) $m[1]; + $size = @filesize($f); + if ($size === false) continue; + + $h = $hosts[$id] ?? ['requests' => 0, 'bytes' => 0, 's2xx' => 0, 's3xx' => 0, + 's4xx' => 0, 's5xx' => 0, 'offset' => 0, 'last_seen' => null, + 'rotations' => 0, 'since' => time()]; + $off = (int) ($h['offset'] ?? 0); + + // Smaller than where we stopped means the file was rotated out from under us. Counters are + // cumulative and stay; only the offset resets, and the rotation is counted so the store can + // say the totals have a gap in them. + if ($size < $off) { $off = 0; $h['rotations'] = ($h['rotations'] ?? 0) + 1; } + if ($size === $off) { $hosts[$id] = $h; continue; } + + $fp = @fopen($f, 'rb'); + if (!$fp) { $hosts[$id] = $h; continue; } + @fseek($fp, $off); + + $budget = VV_NPM_MAX_BYTES; + $read = 0; + $lastCompleteOffset = $off; + + while (!feof($fp) && $budget > 0) { + $line = fgets($fp, 8192); + if ($line === false) break; + $len = strlen($line); + $budget -= $len; + $read += $len; + // No trailing newline means nginx is mid-write. Stop and leave the offset before it. + if (substr($line, -1) !== "\n") break; + $lastCompleteOffset += $len; + + // [09/Aug/2026:05:52:08 +0000] - 200 200 - GET https host "/" [Client 1.2.3.4] [Length 567] ... + if (!preg_match('/^\[([^\]]+)\]\s+\S+\s+(\d{3})/', $line, $lm)) continue; + $code = (int) $lm[2]; + $h['requests']++; + $newLines++; + if ($code >= 500) $h['s5xx']++; + elseif ($code >= 400) $h['s4xx']++; + elseif ($code >= 300) $h['s3xx']++; + elseif ($code >= 200) $h['s2xx']++; + if (preg_match('/\[Length (\d+)\]/', $line, $bm)) $h['bytes'] += (int) $bm[1]; + // The log stamp is nginx's own format; a line that will not parse is not worth a + // guessed timestamp, so last_seen simply does not move for it. + $ts = strtotime(str_replace('/', ' ', preg_replace('/^(\d+)\/(\w+)\/(\d+):/', '$1 $2 $3 ', $lm[1]))); + if ($ts !== false && ($h['last_seen'] === null || $ts > $h['last_seen'])) $h['last_seen'] = $ts; + } + fclose($fp); + + $h['offset'] = $lastCompleteOffset; + $hosts[$id] = $h; + $readTotal += $read; + } + + ksort($hosts, SORT_NUMERIC); + $store['hosts'] = $hosts; + $store['last_pass'] = time(); + + printf("%d logs, read %s this pass, %s new requests, %.1fs\n", + count($files), vv_npm_bytes($readTotal), number_format($newLines), microtime(true) - $t0); + + if ($dryRun) { echo "dry run — nothing written\n"; exit(0); } + if (!vv_npm_stats_write($store)) { echo 'could not write ' . vv_npm_stats_path() . "\n"; exit(1); } + echo 'wrote ' . vv_npm_stats_path() . "\n"; + exit(0); + +} finally { + flock($lock, LOCK_UN); + fclose($lock); +} diff --git a/Plugin/unraid/Tools/npm_access_stats.sh b/Plugin/unraid/Tools/npm_access_stats.sh new file mode 100755 index 0000000..eedde8c --- /dev/null +++ b/Plugin/unraid/Tools/npm_access_stats.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# ============================================================================================== +# ================================ NPM Access Stats ============================================ +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Aggregates Nginx Proxy Manager's per-host access logs into DB_DIR/npm_access.json — requests, +# bytes sent, status breakdown and last hit per proxy host. The Proxies tab reads it. +# +# NPM writes one access log per host and counts nothing. The logs here are 475 MB across 41 files, +# so this cannot happen inside a page load; each pass reads only what arrived since the last one. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# A wrapper. The work is in npm_access_stats.php, next to the NPM client and conf helpers it uses. +# Same split as api_cache_writer, ai_repair_sweep and cert_history. +# +# Totals are "since tracking began", not since the host existed: lines that rotate out between two +# passes are not counted. Running daily keeps that to whatever NPM rotates in a day. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# npm_access_stats.sh one pass +# npm_access_stats.sh --dry-run parse and report, write nothing +# npm_access_stats.sh --status print the store +# npm_access_stats.sh --reset forget offsets and totals, start again from the current logs +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# NPM_LOG_DIR override the log directory; otherwise derived from the container's mounts +# DB_DIR npm_access.json is written here +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +php "$SCRIPT_DIR/npm_access_stats.php" "$@" diff --git a/Plugin/unraid/api/auth.php b/Plugin/unraid/api/auth.php index b73fe43..3e585b9 100644 --- a/Plugin/unraid/api/auth.php +++ b/Plugin/unraid/api/auth.php @@ -102,7 +102,7 @@ require_once dirname(__DIR__) . '/include/auth.php'; // panels every stack carries rather than gated to one. const VV_AUTH_ACTION_PANEL = [ // GET - 'npm_proxies' => 'proxies', 'npm_certs' => 'proxies', + 'npm_proxies' => 'proxies', 'npm_certs' => 'proxies', 'npm_stats' => 'proxies', 'lldap_users' => 'users', 'lldap_groups' => 'users', 'lldap_avatar' => 'users', 'authelia_rules' => 'acl', // POST @@ -150,6 +150,17 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') { exit; } + // Per-host request and byte totals, written by Tools/npm_access_stats.sh. Served rather than + // computed: the logs behind these numbers are 475 MB and reading them is a scheduled job, not + // something a page load can do. + if ($action === 'npm_stats') { + $f = rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/npm_access.json'; + $s = is_file($f) ? (json_decode((string) @file_get_contents($f), true) ?: []) : []; + echo json_encode(['ok' => true, 'hosts' => $s['hosts'] ?? [], + 'last_pass' => $s['last_pass'] ?? null]); + exit; + } + $result = match ($action) { 'npm_proxies' => vv_npm_list_proxies(), 'npm_certs' => ['ok' => true, 'certs' => vv_npm_list_certs()], diff --git a/Plugin/unraid/pages/auth.php b/Plugin/unraid/pages/auth.php index 3fb9968..eb07452 100644 --- a/Plugin/unraid/pages/auth.php +++ b/Plugin/unraid/pages/auth.php @@ -239,7 +239,28 @@ require_once dirname(__DIR__) . '/include/ai_chat.php'; /* ── Misc ────────────────────────────────────────────────────────────────── */ .vv-au-empty { padding:24px;text-align:center;font-size:11px;color:#333; } .vv-au-domain { font-size:12px;color:#bbb;font-weight:bold; } -.vv-au-fwd { font-size:10px;color:#444; } +.vv-au-fwd { font-size:10px;color:#444;font-family:monospace;margin-top:1px; } + +/* ── Proxy row marks and traffic ─────────────────────────────────────────── */ +/* Small marks rather than four more columns. Each one is a fact about the host that had no + column at all — most importantly whether an auth_request block is in front of it. */ +.vv-au-marks { display:flex;flex-wrap:wrap;gap:3px;align-items:center; } +.vv-au-m { font-size:9px;padding:1px 5px;border-radius:2px;background:#141414; + border:1px solid #232323;color:#666;white-space:nowrap; } +.vv-au-m.dim { color:#3a3a3a;border-color:#1c1c1c; } +/* The one mark worth finding at a glance: it is the difference between a service the household + can reach and one the whole internet can. */ +.vv-au-m.auth { background:#1a0d2a;border-color:#2a1a4a;color:#9c6ff7;font-weight:600; } +.vv-au-stats { display:flex;gap:8px;align-items:baseline;font-size:10px;white-space:nowrap; } +.vv-au-hits { color:#bbb;font-weight:600;font-size:11px; } +.vv-au-hits.dim { color:#2e2e2e;font-weight:normal; } +.vv-au-sent { color:#4a7a8a; } +.vv-au-err-n { color:#8a7a4a; } +.vv-au-err-n.bad{ color:#ef5350; } +.vv-au-last { color:#3a3a3a;margin-left:auto; } +/* A disabled host is still listed — it is a thing you might re-enable — but it should not read + as part of what is currently serving. */ +.vv-au-off td { opacity:.42; } .vv-au-loading { color:#333;font-size:11px;padding:16px;text-align:center; } /* ── Certs panel ─────────────────────────────────────────────────────────── */ @@ -325,7 +346,7 @@ $tabs = ['proxies' => 'Proxies', 'users' => 'Users & Groups',