From 28273ffc2af617ed2dc0a8adc52d129ffaad9e85 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sun, 16 Aug 2026 12:12:08 -0400 Subject: [PATCH] Retire domains nothing has probed in 90 days, and notice NPM changes within five minutes --- Deployment/master.conf.template | 2 +- Plugin/unraid/Tools/uptime_probe.php | 28 ++++++++++++++++++++++++++-- Plugin/unraid/pages/auth.php | 5 ++++- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 9f4369a..e8a187c 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -1501,7 +1501,7 @@ # request counts. Fifty thousand self-inflicted requests a day would otherwise bury real traffic. UPTIME_PROBE_ENABLED=true # master switch for the per-minute probe UPTIME_PROBE_TIMEOUT=8 # seconds per domain before it counts as down - UPTIME_PROBE_LIST_TTL=900 # seconds to reuse the domain list from NPM before re-reading it + UPTIME_PROBE_LIST_TTL=300 # seconds to reuse the domain list from NPM before re-reading it # ── Auth Sweep ── # Tools/auth_sweep.sh asks the two questions the Auth tab answers about one host, about every host, diff --git a/Plugin/unraid/Tools/uptime_probe.php b/Plugin/unraid/Tools/uptime_probe.php index f5f3082..a756f16 100644 --- a/Plugin/unraid/Tools/uptime_probe.php +++ b/Plugin/unraid/Tools/uptime_probe.php @@ -35,6 +35,9 @@ // samples would be fifty thousand rows a day and the file would be the problem instead of // the answer. Each tier is what one view on the Proxies tab draws: 24h, 7d, 30d, 12 months. // +// Bounded in domains too: a record nothing has probed for VV_DOMAIN_DROP_DAYS is retired, +// so hosts removed or renamed in NPM do not accumulate for ever under the live ones. +// // The domain list follows NPM, not a hand-kept list in conf. // A host added in the Proxies tab starts being probed without anyone remembering to add it // somewhere else. The list is cached so this does not call the NPM API every minute. @@ -54,7 +57,7 @@ // CONFIGURATION // UPTIME_PROBE_ENABLED master switch (default true) // UPTIME_PROBE_TIMEOUT seconds per domain (default 8) -// UPTIME_PROBE_LIST_TTL seconds to reuse the cached domain list (default 900) +// UPTIME_PROBE_LIST_TTL seconds to reuse the cached domain list (default 300) // VV_UPTIME_UA the User-Agent, matched by npm_access_stats.php // ═══════════════════════════════════════════════════════════════════════════════════════════════ @@ -70,6 +73,11 @@ const VV_DAYS_KEEP = 30; // months were bad — not a date lookup, and rolling the day buckets out to 365 would grow the file // twelvefold to answer the same question at a resolution nothing displays. const VV_MONTHS_KEEP = 12; +// A domain removed or renamed in NPM stops being probed, and its record would otherwise sit here +// for ever — the store only ever gained keys before this. 90 days rather than something tighter +// because a host switched off for a season is a normal thing here and its history should survive +// that; a domain nobody has served for a quarter is gone for good. +const VV_DOMAIN_DROP_DAYS = 90; const VV_EVENTS_KEEP = 20; $dryRun = in_array('--dry-run', $argv, true); @@ -177,7 +185,7 @@ try { echo "UPTIME_PROBE_ENABLED is false\n"; exit(0); } $timeout = max(2, (int) ($v['UPTIME_PROBE_TIMEOUT'] ?? 8)); - $listTtl = max(60, (int) ($v['UPTIME_PROBE_LIST_TTL'] ?? 900)); + $listTtl = max(60, (int) ($v['UPTIME_PROBE_LIST_TTL'] ?? 300)); $store = vv_uptime_read(); if (!$store) { echo "uptime.json is malformed — refusing to overwrite it\n"; exit(1); } @@ -295,6 +303,18 @@ try { } curl_multi_close($mh); + // Retire domains nothing has probed in a long time. Keyed on last_at rather than on absence + // from the current list, which matters when NPM is unreachable: the cached list keeps being + // probed, every domain keeps getting a last_at, and an NPM outage therefore cannot empty the + // store. Only a domain that genuinely left the list stops being stamped. + $cutoff = $now - (VV_DOMAIN_DROP_DAYS * 86400); + $dropped = []; + foreach ($doms as $d => $r) { + // A record with no last_at at all is kept. It should not be possible — every probe stamps + // it — and deleting on missing data is the wrong way round for something irreversible. + if (isset($r['last_at']) && $r['last_at'] < $cutoff) { unset($doms[$d]); $dropped[] = $d; } + } + $store['domains'] = $doms; $store['last_pass'] = $now; @@ -302,6 +322,10 @@ try { microtime(true) - $t0, $changes ? '' : ' (no state changes)'); foreach ($changes as $c) echo " $c\n"; foreach ($downList as $c) echo " DOWN $c\n"; + // Named, not silent. Dropping a record throws away months of history, and a line in the log is + // the only trace that it was this and not the store being reset by something else. + foreach ($dropped as $d) + echo " RETIRED $d — not probed in " . VV_DOMAIN_DROP_DAYS . " days\n"; if ($dryRun) { echo "dry run — nothing written\n"; exit(0); } if (!vv_uptime_write($store)) { echo 'could not write ' . vv_uptime_path() . "\n"; exit(1); } diff --git a/Plugin/unraid/pages/auth.php b/Plugin/unraid/pages/auth.php index 2ee6d42..420cee6 100644 --- a/Plugin/unraid/pages/auth.php +++ b/Plugin/unraid/pages/auth.php @@ -1044,8 +1044,11 @@ function _renderUptimeHistory() { const st = stale(d); const dot = st ? '#333' : (rec.state === 'down' ? '#ef5350' : (rec.state === 'up' ? '#2d5a2d' : '#333')); + // "Not probed" rather than "not in NPM": the probe also skips hosts that are merely switched + // off in the Proxies tab, and calling a disabled host deleted would send someone looking for + // something that is still sitting there with its toggle turned off. const dotT = st ? ('last probed ' + (rec.last_at ? _ago(rec.last_at) + ' ago' : 'never') - + ' — no longer in NPM, so nothing is checking it') + + ' — not being probed: disabled or removed in NPM') : (rec.state === 'down' ? 'down now' : (rec.state === 'up' ? 'up now' : 'not yet probed')); const cells = _UP_PERIODS.map(([k]) => { const p = h[k];