Gmer4Lfe
1001c25487
NPM's access log only describes hosts somebody visited; the host most likely to be quietly broken is the one nobody does. Probes carry a User-Agent npm_access_stats.sh drops — unmarked, this monitor would be fifty thousand requests a day in the very logs it reports on.
235 lines
12 KiB
PHP
235 lines
12 KiB
PHP
<?php
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
// PURPOSE
|
|
// Aggregates Nginx Proxy Manager's per-host access logs into a small store the Proxies tab can
|
|
// read: how many requests each host has served, how many bytes went out, how the responses
|
|
// broke down by status, and when it was last hit.
|
|
//
|
|
// WHY IT EXISTS
|
|
// NPM writes one access log per proxy host and nothing that counts them. The logs on this
|
|
// installation are 475 MB across 41 files — one of them 330 MB on its own — so the question
|
|
// "how many times has this site been hit" cannot be answered inside a page load. This runs on a
|
|
// schedule and leaves behind a few kilobytes of JSON.
|
|
//
|
|
// OPERATIONAL MODEL
|
|
// Incremental. Each pass records the byte offset it reached in every log and starts there next
|
|
// time, so the 475 MB is read once and each later pass reads only what has arrived since.
|
|
//
|
|
// Rotation is detected by the file being smaller than the offset already recorded. The counters
|
|
// are cumulative and are never reset by it — but the lines that rotated out between two passes
|
|
// are not counted, so a total is "requests seen since tracking began", not a claim about the
|
|
// whole history of the host. Running daily keeps that gap to whatever NPM rotates in a day.
|
|
//
|
|
// DESIGN PRINCIPLES
|
|
// Read forward, never re-read.
|
|
// fseek to the stored offset and read to the end. Re-parsing a 330 MB log every pass to
|
|
// recompute a number that only grows is the kind of job that quietly becomes the reason a
|
|
// nightly run takes an hour.
|
|
//
|
|
// A partial last line is not counted.
|
|
// nginx is appending while this reads. The offset advances only to the end of the last
|
|
// complete line, so the remainder is picked up whole on the next pass rather than parsed
|
|
// as a truncated record and then parsed again.
|
|
//
|
|
// Totals only. No per-client or per-path breakdown is kept — that is an analytics product, and
|
|
// this exists to answer "is anything using this host, and is it erroring".
|
|
//
|
|
// OPERATIONAL SAFEGUARDS
|
|
// Non-fatal, always: a missing log directory, an unreadable file or an absent NPM exits 0.
|
|
// One pass at a time, flock non-blocking.
|
|
// The store is written tmp + rename and verified before it replaces the previous one.
|
|
// A bounded amount of work per pass — VV_NPM_MAX_BYTES per file — so a log that grew enormously
|
|
// between passes cannot make this run unboundedly long.
|
|
//
|
|
// RUNTIME MODES
|
|
// npm_access_stats.php one pass
|
|
// npm_access_stats.php --dry-run parse and report, write nothing
|
|
// npm_access_stats.php --status print the store
|
|
// npm_access_stats.php --reset forget offsets and totals, start again from the current logs
|
|
//
|
|
// CONFIGURATION
|
|
// NPM_LOG_DIR where NPM's per-host logs live; derived from the container mount when unset
|
|
// DB_DIR npm_access.json is written here
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
require_once dirname(__DIR__) . '/include/auth.php';
|
|
|
|
$dryRun = in_array('--dry-run', $argv, true);
|
|
$status = in_array('--status', $argv, true);
|
|
$reset = in_array('--reset', $argv, true);
|
|
|
|
// 512 MB per file per pass. The first pass over a 330 MB log is the only one that should ever come
|
|
// near it; the cap exists so an unattended run cannot be surprised by a log that exploded.
|
|
const VV_NPM_MAX_BYTES = 536870912;
|
|
|
|
function vv_npm_stats_path(): string {
|
|
return rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/npm_access.json';
|
|
}
|
|
|
|
// The host path to NPM's log directory. Taken from the container's own mount table rather than
|
|
// hardcoded, because that mapping is the thing most likely to differ between installations.
|
|
function vv_npm_log_dir(): string {
|
|
$conf = trim(vv_conf_vars()['NPM_LOG_DIR'] ?? '');
|
|
if ($conf !== '' && is_dir($conf)) return rtrim($conf, '/');
|
|
$name = trim(vv_conf_vars()[strtoupper(vv_detect_host()) . '_NPM_CONTAINER'] ?? 'NginxProxyManager');
|
|
// {{println}}, not a \n escape: the format string is passed through to docker as written, and
|
|
// a backslash-n in a single-quoted PHP string arrives as two literal characters — which is how
|
|
// this silently found no mounts and reported the log directory missing.
|
|
$out = shell_exec('docker inspect ' . escapeshellarg($name)
|
|
. ' --format ' . escapeshellarg('{{range .Mounts}}{{println .Source ":" .Destination}}{{end}}')
|
|
. ' 2>/dev/null');
|
|
foreach (explode("\n", trim((string) $out)) as $line) {
|
|
// println space-separates its arguments, so the mapping arrives as "src : dst".
|
|
$line = str_replace(' : ', ':', trim($line));
|
|
[$src, $dst] = array_pad(explode(':', $line, 2), 2, '');
|
|
if ($dst === '/config' && $src !== '' && is_dir("$src/log")) return "$src/log";
|
|
if ($dst === '/data' && $src !== '' && is_dir("$src/logs")) return "$src/logs";
|
|
}
|
|
return '';
|
|
}
|
|
|
|
function vv_npm_stats_read(): array {
|
|
$p = vv_npm_stats_path();
|
|
if (!is_file($p)) return ['hosts' => []];
|
|
$j = json_decode((string) @file_get_contents($p), true);
|
|
if (!is_array($j) || !isset($j['hosts']) || !is_array($j['hosts'])) return [];
|
|
return $j;
|
|
}
|
|
|
|
function vv_npm_stats_write(array $data): bool {
|
|
$p = vv_npm_stats_path();
|
|
if (!is_dir(dirname($p)) && !@mkdir(dirname($p), 0755, true)) return false;
|
|
$data['updated'] = time();
|
|
$json = json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
|
|
if ($json === false) return false;
|
|
$tmp = $p . '.vv.tmp';
|
|
if (@file_put_contents($tmp, $json) === false) return false;
|
|
if (json_decode((string) @file_get_contents($tmp), true) === null) { @unlink($tmp); return false; }
|
|
if (!@rename($tmp, $p)) { @unlink($tmp); return false; }
|
|
return true;
|
|
}
|
|
|
|
// ── Status ────────────────────────────────────────────────────────────────────
|
|
if ($status) {
|
|
$s = vv_npm_stats_read();
|
|
if (!$s) { echo "npm_access.json is unreadable\n"; exit(0); }
|
|
printf("%-6s %12s %12s %8s %8s %8s %s\n", 'host', 'requests', 'sent', '2xx', '4xx', '5xx', 'last hit');
|
|
foreach ($s['hosts'] ?? [] as $id => $h) {
|
|
printf("%-6s %12s %12s %8s %8s %8s %s\n", $id,
|
|
number_format($h['requests'] ?? 0), vv_npm_bytes($h['bytes'] ?? 0),
|
|
number_format($h['s2xx'] ?? 0), number_format($h['s4xx'] ?? 0),
|
|
number_format($h['s5xx'] ?? 0),
|
|
!empty($h['last_seen']) ? date('Y-m-d H:i', $h['last_seen']) : '-');
|
|
}
|
|
exit(0);
|
|
}
|
|
|
|
// Decimal, because this is a quantity of traffic and every network tool that will be compared
|
|
// against it is decimal too. Memory is the binary one.
|
|
function vv_npm_bytes(float $b): string {
|
|
$u = ['B', 'kB', 'MB', 'GB', 'TB'];
|
|
$i = 0;
|
|
while ($b >= 1000 && $i < count($u) - 1) { $b /= 1000; $i++; }
|
|
return round($b, $b < 10 && $i ? 1 : 0) . ' ' . $u[$i];
|
|
}
|
|
|
|
$lock = @fopen(sys_get_temp_dir() . '/vv_npm_access.lock', 'c');
|
|
if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) { echo "another pass is running\n"; exit(0); }
|
|
|
|
try {
|
|
$dir = vv_npm_log_dir();
|
|
if ($dir === '') { echo "NPM log directory not found — set NPM_LOG_DIR\n"; exit(0); }
|
|
|
|
$store = $reset ? ['hosts' => []] : vv_npm_stats_read();
|
|
if (!$store) { echo "npm_access.json is malformed — refusing to overwrite it\n"; exit(1); }
|
|
$hosts = $store['hosts'] ?? [];
|
|
|
|
$files = glob("$dir/proxy-host-*_access.log") ?: [];
|
|
if (!$files) { echo "no per-host access logs in $dir\n"; exit(0); }
|
|
|
|
$t0 = microtime(true);
|
|
$readTotal = 0; $newLines = 0;
|
|
$skipped = 0;
|
|
|
|
foreach ($files as $f) {
|
|
if (!preg_match('/proxy-host-(\d+)_access\.log$/', $f, $m)) continue;
|
|
$id = (string) (int) $m[1];
|
|
$size = @filesize($f);
|
|
if ($size === false) continue;
|
|
|
|
$h = $hosts[$id] ?? ['requests' => 0, 'bytes' => 0, 's2xx' => 0, 's3xx' => 0,
|
|
's4xx' => 0, 's5xx' => 0, 'offset' => 0, 'last_seen' => null,
|
|
'rotations' => 0, 'since' => time()];
|
|
$off = (int) ($h['offset'] ?? 0);
|
|
|
|
// Smaller than where we stopped means the file was rotated out from under us. Counters are
|
|
// cumulative and stay; only the offset resets, and the rotation is counted so the store can
|
|
// say the totals have a gap in them.
|
|
if ($size < $off) { $off = 0; $h['rotations'] = ($h['rotations'] ?? 0) + 1; }
|
|
if ($size === $off) { $hosts[$id] = $h; continue; }
|
|
|
|
$fp = @fopen($f, 'rb');
|
|
if (!$fp) { $hosts[$id] = $h; continue; }
|
|
@fseek($fp, $off);
|
|
|
|
$budget = VV_NPM_MAX_BYTES;
|
|
$read = 0;
|
|
$lastCompleteOffset = $off;
|
|
|
|
while (!feof($fp) && $budget > 0) {
|
|
$line = fgets($fp, 8192);
|
|
if ($line === false) break;
|
|
$len = strlen($line);
|
|
$budget -= $len;
|
|
$read += $len;
|
|
// No trailing newline means nginx is mid-write. Stop and leave the offset before it.
|
|
if (substr($line, -1) !== "\n") break;
|
|
$lastCompleteOffset += $len;
|
|
|
|
// Varaverk's own uptime probe, dropped before it is counted. It hits every host once a
|
|
// minute — fifty thousand requests a day — so counting it would make this monitor the
|
|
// overwhelming majority of the traffic it reports, and a host nobody visits would look
|
|
// as busy as one that is genuinely used. The string is VV_UPTIME_UA in uptime_probe.php
|
|
// and the two must stay in step.
|
|
if (strpos($line, 'Varaverk-Uptime/') !== false) { $skipped++; continue; }
|
|
|
|
// [09/Aug/2026:05:52:08 +0000] - 200 200 - GET https host "/" [Client 1.2.3.4] [Length 567] ...
|
|
if (!preg_match('/^\[([^\]]+)\]\s+\S+\s+(\d{3})/', $line, $lm)) continue;
|
|
$code = (int) $lm[2];
|
|
$h['requests']++;
|
|
$newLines++;
|
|
if ($code >= 500) $h['s5xx']++;
|
|
elseif ($code >= 400) $h['s4xx']++;
|
|
elseif ($code >= 300) $h['s3xx']++;
|
|
elseif ($code >= 200) $h['s2xx']++;
|
|
if (preg_match('/\[Length (\d+)\]/', $line, $bm)) $h['bytes'] += (int) $bm[1];
|
|
// The log stamp is nginx's own format; a line that will not parse is not worth a
|
|
// guessed timestamp, so last_seen simply does not move for it.
|
|
$ts = strtotime(str_replace('/', ' ', preg_replace('/^(\d+)\/(\w+)\/(\d+):/', '$1 $2 $3 ', $lm[1])));
|
|
if ($ts !== false && ($h['last_seen'] === null || $ts > $h['last_seen'])) $h['last_seen'] = $ts;
|
|
}
|
|
fclose($fp);
|
|
|
|
$h['offset'] = $lastCompleteOffset;
|
|
$hosts[$id] = $h;
|
|
$readTotal += $read;
|
|
}
|
|
|
|
ksort($hosts, SORT_NUMERIC);
|
|
$store['hosts'] = $hosts;
|
|
$store['last_pass'] = time();
|
|
|
|
printf("%d logs, read %s this pass, %s new requests, %s own probes ignored, %.1fs\n",
|
|
count($files), vv_npm_bytes($readTotal), number_format($newLines),
|
|
number_format($skipped), microtime(true) - $t0);
|
|
|
|
if ($dryRun) { echo "dry run — nothing written\n"; exit(0); }
|
|
if (!vv_npm_stats_write($store)) { echo 'could not write ' . vv_npm_stats_path() . "\n"; exit(1); }
|
|
echo 'wrote ' . vv_npm_stats_path() . "\n";
|
|
exit(0);
|
|
|
|
} finally {
|
|
flock($lock, LOCK_UN);
|
|
fclose($lock);
|
|
}
|