From 1041c4bba898887d20c151799324887b7f3b1190 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sat, 15 Aug 2026 19:20:11 -0400 Subject: [PATCH] Show what each proxy host is actually doing, and mark the ones behind Authelia NPM writes an access log per host and counts nothing, so 475 MB of logs held the only answer to "is anything using this". Aggregated on a schedule and read from a few kB of JSON; the row also now says whether an auth_request block is in front of the site, which nothing showed before. --- Deployment/master.conf.template | 1 + Plugin/unraid/Tools/npm_access_stats.php | 225 +++++++++++++++++++++++ Plugin/unraid/Tools/npm_access_stats.sh | 43 +++++ Plugin/unraid/api/auth.php | 13 +- Plugin/unraid/pages/auth.php | 104 ++++++++++- 5 files changed, 376 insertions(+), 10 deletions(-) create mode 100644 Plugin/unraid/Tools/npm_access_stats.php create mode 100755 Plugin/unraid/Tools/npm_access_stats.sh diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 31ca137..7debb44 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -493,6 +493,7 @@ # make five strikes five weeks. One NPM list call; it is what gives the Certs tab its # per-domain renewal and failure counts. "Plugin/unraid/Tools/cert_history.sh" # record cert renewals, failures and age per domain + "Plugin/unraid/Tools/npm_access_stats.sh" # aggregate NPM per-host access logs into request and byte totals ) # Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart. diff --git a/Plugin/unraid/Tools/npm_access_stats.php b/Plugin/unraid/Tools/npm_access_stats.php new file mode 100644 index 0000000..00ae377 --- /dev/null +++ b/Plugin/unraid/Tools/npm_access_stats.php @@ -0,0 +1,225 @@ +/dev/null'); + foreach (explode("\n", trim((string) $out)) as $line) { + // println space-separates its arguments, so the mapping arrives as "src : dst". + $line = str_replace(' : ', ':', trim($line)); + [$src, $dst] = array_pad(explode(':', $line, 2), 2, ''); + if ($dst === '/config' && $src !== '' && is_dir("$src/log")) return "$src/log"; + if ($dst === '/data' && $src !== '' && is_dir("$src/logs")) return "$src/logs"; + } + return ''; +} + +function vv_npm_stats_read(): array { + $p = vv_npm_stats_path(); + if (!is_file($p)) return ['hosts' => []]; + $j = json_decode((string) @file_get_contents($p), true); + if (!is_array($j) || !isset($j['hosts']) || !is_array($j['hosts'])) return []; + return $j; +} + +function vv_npm_stats_write(array $data): bool { + $p = vv_npm_stats_path(); + if (!is_dir(dirname($p)) && !@mkdir(dirname($p), 0755, true)) return false; + $data['updated'] = time(); + $json = json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + if ($json === false) return false; + $tmp = $p . '.vv.tmp'; + if (@file_put_contents($tmp, $json) === false) return false; + if (json_decode((string) @file_get_contents($tmp), true) === null) { @unlink($tmp); return false; } + if (!@rename($tmp, $p)) { @unlink($tmp); return false; } + return true; +} + +// ── Status ──────────────────────────────────────────────────────────────────── +if ($status) { + $s = vv_npm_stats_read(); + if (!$s) { echo "npm_access.json is unreadable\n"; exit(0); } + printf("%-6s %12s %12s %8s %8s %8s %s\n", 'host', 'requests', 'sent', '2xx', '4xx', '5xx', 'last hit'); + foreach ($s['hosts'] ?? [] as $id => $h) { + printf("%-6s %12s %12s %8s %8s %8s %s\n", $id, + number_format($h['requests'] ?? 0), vv_npm_bytes($h['bytes'] ?? 0), + number_format($h['s2xx'] ?? 0), number_format($h['s4xx'] ?? 0), + number_format($h['s5xx'] ?? 0), + !empty($h['last_seen']) ? date('Y-m-d H:i', $h['last_seen']) : '-'); + } + exit(0); +} + +// Decimal, because this is a quantity of traffic and every network tool that will be compared +// against it is decimal too. Memory is the binary one. +function vv_npm_bytes(float $b): string { + $u = ['B', 'kB', 'MB', 'GB', 'TB']; + $i = 0; + while ($b >= 1000 && $i < count($u) - 1) { $b /= 1000; $i++; } + return round($b, $b < 10 && $i ? 1 : 0) . ' ' . $u[$i]; +} + +$lock = @fopen(sys_get_temp_dir() . '/vv_npm_access.lock', 'c'); +if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) { echo "another pass is running\n"; exit(0); } + +try { + $dir = vv_npm_log_dir(); + if ($dir === '') { echo "NPM log directory not found — set NPM_LOG_DIR\n"; exit(0); } + + $store = $reset ? ['hosts' => []] : vv_npm_stats_read(); + if (!$store) { echo "npm_access.json is malformed — refusing to overwrite it\n"; exit(1); } + $hosts = $store['hosts'] ?? []; + + $files = glob("$dir/proxy-host-*_access.log") ?: []; + if (!$files) { echo "no per-host access logs in $dir\n"; exit(0); } + + $t0 = microtime(true); + $readTotal = 0; $newLines = 0; + + foreach ($files as $f) { + if (!preg_match('/proxy-host-(\d+)_access\.log$/', $f, $m)) continue; + $id = (string) (int) $m[1]; + $size = @filesize($f); + if ($size === false) continue; + + $h = $hosts[$id] ?? ['requests' => 0, 'bytes' => 0, 's2xx' => 0, 's3xx' => 0, + 's4xx' => 0, 's5xx' => 0, 'offset' => 0, 'last_seen' => null, + 'rotations' => 0, 'since' => time()]; + $off = (int) ($h['offset'] ?? 0); + + // Smaller than where we stopped means the file was rotated out from under us. Counters are + // cumulative and stay; only the offset resets, and the rotation is counted so the store can + // say the totals have a gap in them. + if ($size < $off) { $off = 0; $h['rotations'] = ($h['rotations'] ?? 0) + 1; } + if ($size === $off) { $hosts[$id] = $h; continue; } + + $fp = @fopen($f, 'rb'); + if (!$fp) { $hosts[$id] = $h; continue; } + @fseek($fp, $off); + + $budget = VV_NPM_MAX_BYTES; + $read = 0; + $lastCompleteOffset = $off; + + while (!feof($fp) && $budget > 0) { + $line = fgets($fp, 8192); + if ($line === false) break; + $len = strlen($line); + $budget -= $len; + $read += $len; + // No trailing newline means nginx is mid-write. Stop and leave the offset before it. + if (substr($line, -1) !== "\n") break; + $lastCompleteOffset += $len; + + // [09/Aug/2026:05:52:08 +0000] - 200 200 - GET https host "/" [Client 1.2.3.4] [Length 567] ... + if (!preg_match('/^\[([^\]]+)\]\s+\S+\s+(\d{3})/', $line, $lm)) continue; + $code = (int) $lm[2]; + $h['requests']++; + $newLines++; + if ($code >= 500) $h['s5xx']++; + elseif ($code >= 400) $h['s4xx']++; + elseif ($code >= 300) $h['s3xx']++; + elseif ($code >= 200) $h['s2xx']++; + if (preg_match('/\[Length (\d+)\]/', $line, $bm)) $h['bytes'] += (int) $bm[1]; + // The log stamp is nginx's own format; a line that will not parse is not worth a + // guessed timestamp, so last_seen simply does not move for it. + $ts = strtotime(str_replace('/', ' ', preg_replace('/^(\d+)\/(\w+)\/(\d+):/', '$1 $2 $3 ', $lm[1]))); + if ($ts !== false && ($h['last_seen'] === null || $ts > $h['last_seen'])) $h['last_seen'] = $ts; + } + fclose($fp); + + $h['offset'] = $lastCompleteOffset; + $hosts[$id] = $h; + $readTotal += $read; + } + + ksort($hosts, SORT_NUMERIC); + $store['hosts'] = $hosts; + $store['last_pass'] = time(); + + printf("%d logs, read %s this pass, %s new requests, %.1fs\n", + count($files), vv_npm_bytes($readTotal), number_format($newLines), microtime(true) - $t0); + + if ($dryRun) { echo "dry run — nothing written\n"; exit(0); } + if (!vv_npm_stats_write($store)) { echo 'could not write ' . vv_npm_stats_path() . "\n"; exit(1); } + echo 'wrote ' . vv_npm_stats_path() . "\n"; + exit(0); + +} finally { + flock($lock, LOCK_UN); + fclose($lock); +} diff --git a/Plugin/unraid/Tools/npm_access_stats.sh b/Plugin/unraid/Tools/npm_access_stats.sh new file mode 100755 index 0000000..eedde8c --- /dev/null +++ b/Plugin/unraid/Tools/npm_access_stats.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# ============================================================================================== +# ================================ NPM Access Stats ============================================ +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Aggregates Nginx Proxy Manager's per-host access logs into DB_DIR/npm_access.json — requests, +# bytes sent, status breakdown and last hit per proxy host. The Proxies tab reads it. +# +# NPM writes one access log per host and counts nothing. The logs here are 475 MB across 41 files, +# so this cannot happen inside a page load; each pass reads only what arrived since the last one. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# A wrapper. The work is in npm_access_stats.php, next to the NPM client and conf helpers it uses. +# Same split as api_cache_writer, ai_repair_sweep and cert_history. +# +# Totals are "since tracking began", not since the host existed: lines that rotate out between two +# passes are not counted. Running daily keeps that to whatever NPM rotates in a day. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# npm_access_stats.sh one pass +# npm_access_stats.sh --dry-run parse and report, write nothing +# npm_access_stats.sh --status print the store +# npm_access_stats.sh --reset forget offsets and totals, start again from the current logs +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# NPM_LOG_DIR override the log directory; otherwise derived from the container's mounts +# DB_DIR npm_access.json is written here +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +php "$SCRIPT_DIR/npm_access_stats.php" "$@" diff --git a/Plugin/unraid/api/auth.php b/Plugin/unraid/api/auth.php index b73fe43..3e585b9 100644 --- a/Plugin/unraid/api/auth.php +++ b/Plugin/unraid/api/auth.php @@ -102,7 +102,7 @@ require_once dirname(__DIR__) . '/include/auth.php'; // panels every stack carries rather than gated to one. const VV_AUTH_ACTION_PANEL = [ // GET - 'npm_proxies' => 'proxies', 'npm_certs' => 'proxies', + 'npm_proxies' => 'proxies', 'npm_certs' => 'proxies', 'npm_stats' => 'proxies', 'lldap_users' => 'users', 'lldap_groups' => 'users', 'lldap_avatar' => 'users', 'authelia_rules' => 'acl', // POST @@ -150,6 +150,17 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') { exit; } + // Per-host request and byte totals, written by Tools/npm_access_stats.sh. Served rather than + // computed: the logs behind these numbers are 475 MB and reading them is a scheduled job, not + // something a page load can do. + if ($action === 'npm_stats') { + $f = rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/npm_access.json'; + $s = is_file($f) ? (json_decode((string) @file_get_contents($f), true) ?: []) : []; + echo json_encode(['ok' => true, 'hosts' => $s['hosts'] ?? [], + 'last_pass' => $s['last_pass'] ?? null]); + exit; + } + $result = match ($action) { 'npm_proxies' => vv_npm_list_proxies(), 'npm_certs' => ['ok' => true, 'certs' => vv_npm_list_certs()], diff --git a/Plugin/unraid/pages/auth.php b/Plugin/unraid/pages/auth.php index 3fb9968..eb07452 100644 --- a/Plugin/unraid/pages/auth.php +++ b/Plugin/unraid/pages/auth.php @@ -239,7 +239,28 @@ require_once dirname(__DIR__) . '/include/ai_chat.php'; /* ── Misc ────────────────────────────────────────────────────────────────── */ .vv-au-empty { padding:24px;text-align:center;font-size:11px;color:#333; } .vv-au-domain { font-size:12px;color:#bbb;font-weight:bold; } -.vv-au-fwd { font-size:10px;color:#444; } +.vv-au-fwd { font-size:10px;color:#444;font-family:monospace;margin-top:1px; } + +/* ── Proxy row marks and traffic ─────────────────────────────────────────── */ +/* Small marks rather than four more columns. Each one is a fact about the host that had no + column at all — most importantly whether an auth_request block is in front of it. */ +.vv-au-marks { display:flex;flex-wrap:wrap;gap:3px;align-items:center; } +.vv-au-m { font-size:9px;padding:1px 5px;border-radius:2px;background:#141414; + border:1px solid #232323;color:#666;white-space:nowrap; } +.vv-au-m.dim { color:#3a3a3a;border-color:#1c1c1c; } +/* The one mark worth finding at a glance: it is the difference between a service the household + can reach and one the whole internet can. */ +.vv-au-m.auth { background:#1a0d2a;border-color:#2a1a4a;color:#9c6ff7;font-weight:600; } +.vv-au-stats { display:flex;gap:8px;align-items:baseline;font-size:10px;white-space:nowrap; } +.vv-au-hits { color:#bbb;font-weight:600;font-size:11px; } +.vv-au-hits.dim { color:#2e2e2e;font-weight:normal; } +.vv-au-sent { color:#4a7a8a; } +.vv-au-err-n { color:#8a7a4a; } +.vv-au-err-n.bad{ color:#ef5350; } +.vv-au-last { color:#3a3a3a;margin-left:auto; } +/* A disabled host is still listed — it is a thing you might re-enable — but it should not read + as part of what is currently serving. */ +.vv-au-off td { opacity:.42; } .vv-au-loading { color:#333;font-size:11px;padding:16px;text-align:center; } /* ── Certs panel ─────────────────────────────────────────────────────────── */ @@ -325,7 +346,7 @@ $tabs = ['proxies' => 'Proxies', 'users' => 'Users & Groups',
Loading…
- + @@ -479,6 +500,8 @@ const IS_OWNER = ; // ── State ───────────────────────────────────────────────────────────────────── let _proxies = [], _certs = []; +// Keyed by proxy-host id, from Tools/npm_access_stats.sh. Empty until it has run once. +let _proxyStats = {}; let _users = [], _groups = []; let _rules = [], _defaultPolicy = 'deny'; // The trailing comment on the default_policy line, carried so a save puts it back. The block is @@ -613,6 +636,9 @@ function _loadProxies() { _renderProxies(); } + // Stats are optional decoration — the list must render whether or not the aggregator has run, + // so this neither blocks _check() nor fails the load. + _get('npm_stats', r => { _proxyStats = (r && r.ok && r.hosts) ? r.hosts : {}; if (proxiesLoaded) _renderProxies(); }); _get('npm_certs', r => { _certs = r.certs || []; certsLoaded = true; _check(); }); _get('npm_proxies', r => { if (!r.ok) { loading.innerHTML = ''+_esc(r.error)+''; return; } @@ -631,14 +657,43 @@ function _renderProxies() { const domains = (p.domain_names || []).join(', '); const fwd = p.forward_scheme + '://' + p.forward_host + ':' + p.forward_port; const hasSsl = p.certificate_id && p.certificate_id !== '0'; - const sslBadge = hasSsl - ? `SSL` - : `none`; const enabled = p.enabled; - return ` -
${_esc(domains)}
-
${_esc(fwd)}
- ${sslBadge} + const st = _proxyStats[String(p.id)] || null; + + // Everything the host is doing that used to be invisible, as one row of small marks rather + // than four more columns: whether it is protected by an auth_request block, whether SSL is + // forced, and whether it carries custom nginx at all. 25 of these have config the page could + // not previously show, and the auth ones are the important case. + const adv = (p.advanced_config || '').trim(); + const guarded = /auth_request/.test(adv); + const marks = [ + hasSsl ? `SSL` + : `none`, + p.ssl_forced ? 'force' : '', + guarded ? 'auth' : '', + adv && !guarded ? 'nginx' : '', + p.http2_support ? 'h2' : '', + p.hsts_enabled ? 'hsts' : '', + p.block_exploits ? 'blk' : '', + ].filter(Boolean).join(''); + + // Only shown once the aggregator has run. A dash is honest; a zero would read as "nobody has + // ever visited this" when it means "nothing has counted yet". + const hits = st ? `${_fmtNum(st.requests)}` : '—'; + const sent = st ? `${_fmtBytes(st.bytes)}` : ''; + const errs = (st && (st.s4xx || st.s5xx)) + ? `${_fmtNum(st.s4xx + st.s5xx)} err` + : ''; + const last = (st && st.last_seen) + ? `${_ago(st.last_seen)}` : ''; + + return ` + +
${_esc(domains)}
+
→ ${_esc(fwd)}
+ +
${marks}
+
${hits}${sent}${errs}${last}
${_togHtml('proxy-' + p.id, enabled, enabled ? 'Enabled — click to disable' : 'Disabled — click to enable')}
@@ -650,6 +705,37 @@ function _renderProxies() { }).join(''); } +function _certName(id) { + const c = _certs.find(x => String(x.id) === String(id)); + return c ? (c.nice_name || (c.domain_names || []).join(', ')) : 'certificate ' + id; +} + +// Compact because these sit in a column beside a domain name: 664,673 is wider than the name it +// belongs to and the exact figure is in the title attribute either way. +function _fmtNum(n) { + n = Number(n) || 0; + if (n >= 1e6) return (n / 1e6).toFixed(n < 1e7 ? 1 : 0) + 'M'; + if (n >= 1e3) return (n / 1e3).toFixed(n < 1e4 ? 1 : 0) + 'k'; + return String(n); +} + +// Decimal, matching every network tool this will be compared against. Memory is the binary one. +function _fmtBytes(b) { + b = Number(b) || 0; + const u = ['B','kB','MB','GB','TB']; + let i = 0; + while (b >= 1000 && i < u.length - 1) { b /= 1000; i++; } + return (b < 10 && i ? b.toFixed(1) : Math.round(b)) + ' ' + u[i]; +} + +function _ago(ts) { + const s = Math.max(0, Math.floor(Date.now() / 1000 - ts)); + if (s < 90) return s + 's'; + if (s < 5400) return Math.round(s / 60) + 'm'; + if (s < 172800)return Math.round(s / 3600) + 'h'; + return Math.round(s / 86400) + 'd'; +} + function _proxyModal(id) { const p = id ? _proxies.find(x => x.id === id) : null; const certOptions = _certs.map(c =>