Probe every domain every minute, and keep it out of its own traffic numbers

NPM's access log only describes hosts somebody visited; the host most likely to be quietly broken
is the one nobody does. Probes carry a User-Agent npm_access_stats.sh drops — unmarked, this
monitor would be fifty thousand requests a day in the very logs it reports on.
This commit is contained in:
Gmer4Lfe
2026-08-15 20:28:30 -04:00
parent c0ace5a0ca
commit 1001c25487
9 changed files with 534 additions and 5 deletions
+11 -2
View File
@@ -149,6 +149,7 @@ try {
$t0 = microtime(true);
$readTotal = 0; $newLines = 0;
$skipped = 0;
foreach ($files as $f) {
if (!preg_match('/proxy-host-(\d+)_access\.log$/', $f, $m)) continue;
@@ -185,6 +186,13 @@ try {
if (substr($line, -1) !== "\n") break;
$lastCompleteOffset += $len;
// Varaverk's own uptime probe, dropped before it is counted. It hits every host once a
// minute — fifty thousand requests a day — so counting it would make this monitor the
// overwhelming majority of the traffic it reports, and a host nobody visits would look
// as busy as one that is genuinely used. The string is VV_UPTIME_UA in uptime_probe.php
// and the two must stay in step.
if (strpos($line, 'Varaverk-Uptime/') !== false) { $skipped++; continue; }
// [09/Aug/2026:05:52:08 +0000] - 200 200 - GET https host "/" [Client 1.2.3.4] [Length 567] ...
if (!preg_match('/^\[([^\]]+)\]\s+\S+\s+(\d{3})/', $line, $lm)) continue;
$code = (int) $lm[2];
@@ -211,8 +219,9 @@ try {
$store['hosts'] = $hosts;
$store['last_pass'] = time();
printf("%d logs, read %s this pass, %s new requests, %.1fs\n",
count($files), vv_npm_bytes($readTotal), number_format($newLines), microtime(true) - $t0);
printf("%d logs, read %s this pass, %s new requests, %s own probes ignored, %.1fs\n",
count($files), vv_npm_bytes($readTotal), number_format($newLines),
number_format($skipped), microtime(true) - $t0);
if ($dryRun) { echo "dry run — nothing written\n"; exit(0); }
if (!vv_npm_stats_write($store)) { echo 'could not write ' . vv_npm_stats_path() . "\n"; exit(1); }