338 lines
17 KiB
PHP
338 lines
17 KiB
PHP
<?php
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
// PURPOSE
|
|
// Probes every hostname NPM serves, from outside the proxy, and records whether it answered.
|
|
// Produces the uptime percentages and the recent-history strip on the Proxies tab, and the list
|
|
// of anything that had a wobble for the Sunday report.
|
|
//
|
|
// WHY IT EXISTS
|
|
// NPM's access log says what happened when somebody visited. It says nothing at all about a host
|
|
// nobody visited, which is exactly the host most likely to be quietly broken. Five hosts here
|
|
// have been returning errors on every request for months without anything noticing.
|
|
//
|
|
// OPERATIONAL MODEL
|
|
// One pass = one sample per domain, all issued in parallel through curl_multi so the wall time
|
|
// is roughly the slowest domain rather than the sum of all of them. Intended to run every
|
|
// minute; it is safe at any interval because everything is stored as counts and buckets rather
|
|
// than assuming a cadence.
|
|
//
|
|
// up = the host answered with an HTTP status below 500.
|
|
// down = nothing answered, the TLS handshake failed, or it answered 5xx.
|
|
//
|
|
// A 401 or a 302 to the Authelia portal is UP. The question this asks is "is the server there",
|
|
// and an auth redirect is the strongest possible evidence that it is. Counting a protected site
|
|
// as down would mark every guarded host on this installation permanently offline.
|
|
//
|
|
// DESIGN PRINCIPLES
|
|
// Probes carry a User-Agent that the access-log aggregator ignores.
|
|
// Thirty-five domains once a minute is fifty thousand requests a day landing in the very
|
|
// logs Tools/npm_access_stats.sh counts. Left unmarked, this monitor would become the
|
|
// majority of the traffic it reports and bury whatever real use these hosts get.
|
|
//
|
|
// Bounded storage, decided up front.
|
|
// Per domain: lifetime counters, the last 60 samples for the strip, 48 hourly buckets,
|
|
// 30 daily ones and 12 monthly ones. That is a fixed size — a per-minute probe kept as raw
|
|
// samples would be fifty thousand rows a day and the file would be the problem instead of
|
|
// the answer. Each tier is what one view on the Proxies tab draws: 24h, 7d, 30d, 12 months.
|
|
//
|
|
// Bounded in domains too: a record nothing has probed for VV_DOMAIN_DROP_DAYS is retired,
|
|
// so hosts removed or renamed in NPM do not accumulate for ever under the live ones.
|
|
//
|
|
// The domain list follows NPM, not a hand-kept list in conf.
|
|
// A host added in the Proxies tab starts being probed without anyone remembering to add it
|
|
// somewhere else. The list is cached so this does not call the NPM API every minute.
|
|
//
|
|
// OPERATIONAL SAFEGUARDS
|
|
// Non-fatal, always: no NPM, no credentials, no domains — exits 0.
|
|
// One pass at a time, flock non-blocking, so a slow pass cannot overlap the next minute's.
|
|
// Store written tmp + rename, verified before it replaces the previous file.
|
|
// HEAD, not GET, and nothing is followed — this is a liveness check, not a crawler.
|
|
//
|
|
// RUNTIME MODES
|
|
// uptime_probe.php one pass
|
|
// uptime_probe.php --dry-run probe and report, write nothing
|
|
// uptime_probe.php --status per-domain uptime table
|
|
// uptime_probe.php --events recent state changes, newest first
|
|
//
|
|
// CONFIGURATION
|
|
// UPTIME_PROBE_ENABLED master switch (default true)
|
|
// UPTIME_PROBE_TIMEOUT seconds per domain (default 8)
|
|
// UPTIME_PROBE_LIST_TTL seconds to reuse the cached domain list (default 300)
|
|
// VV_UPTIME_UA the User-Agent, matched by npm_access_stats.php
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
require_once dirname(__DIR__) . '/include/auth.php';
|
|
|
|
// Shared with npm_access_stats.php, which drops any log line containing it. Changing this in one
|
|
// place and not the other turns the monitor's own traffic back into counted requests.
|
|
const VV_UPTIME_UA = 'Varaverk-Uptime/1.0';
|
|
const VV_SAMPLES_KEEP = 60; // one hour at a one-minute cadence
|
|
const VV_HOURS_KEEP = 48;
|
|
const VV_DAYS_KEEP = 30;
|
|
// A year as twelve monthly buckets rather than 365 daily ones. The yearly view is a shape — which
|
|
// months were bad — not a date lookup, and rolling the day buckets out to 365 would grow the file
|
|
// twelvefold to answer the same question at a resolution nothing displays.
|
|
const VV_MONTHS_KEEP = 12;
|
|
// A domain removed or renamed in NPM stops being probed, and its record would otherwise sit here
|
|
// for ever — the store only ever gained keys before this. 90 days rather than something tighter
|
|
// because a host switched off for a season is a normal thing here and its history should survive
|
|
// that; a domain nobody has served for a quarter is gone for good.
|
|
const VV_DOMAIN_DROP_DAYS = 90;
|
|
const VV_EVENTS_KEEP = 20;
|
|
|
|
$dryRun = in_array('--dry-run', $argv, true);
|
|
$status = in_array('--status', $argv, true);
|
|
$events = in_array('--events', $argv, true);
|
|
$report = in_array('--report', $argv, true);
|
|
|
|
function vv_uptime_path(): string {
|
|
return rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/uptime.json';
|
|
}
|
|
|
|
function vv_uptime_read(): array {
|
|
$p = vv_uptime_path();
|
|
if (!is_file($p)) return ['domains' => []];
|
|
$j = json_decode((string) @file_get_contents($p), true);
|
|
if (!is_array($j) || !isset($j['domains']) || !is_array($j['domains'])) return [];
|
|
return $j;
|
|
}
|
|
|
|
function vv_uptime_write(array $d): bool {
|
|
$p = vv_uptime_path();
|
|
if (!is_dir(dirname($p)) && !@mkdir(dirname($p), 0755, true)) return false;
|
|
$d['updated'] = time();
|
|
$json = json_encode($d, JSON_UNESCAPED_SLASHES);
|
|
if ($json === false) return false;
|
|
$tmp = $p . '.vv.tmp';
|
|
if (@file_put_contents($tmp, $json) === false) return false;
|
|
if (json_decode((string) @file_get_contents($tmp), true) === null) { @unlink($tmp); return false; }
|
|
if (!@rename($tmp, $p)) { @unlink($tmp); return false; }
|
|
return true;
|
|
}
|
|
|
|
// ── Report ────────────────────────────────────────────────────────────────────
|
|
// Anything that was not perfect over the last seven days, for the Sunday report. Prints nothing
|
|
// and exits 0 when every domain was clean — the orchestrator's job is to be quiet on a good week,
|
|
// and a report that always says something is a report nobody reads.
|
|
if ($report) {
|
|
$s = vv_uptime_read();
|
|
$doms = $s['domains'] ?? [];
|
|
if (!$doms) { exit(0); }
|
|
$down = $wobble = [];
|
|
foreach ($doms as $d => $r) {
|
|
$w = vv_uptime_window($r['days'] ?? [], 7);
|
|
if (($r['state'] ?? '') === 'down') {
|
|
$since = !empty($r['last_change']) ? ' since ' . date('D H:i', $r['last_change']) : '';
|
|
$down[] = sprintf(' %-34s DOWN%s — %s', $d, $since, $r['last_detail'] ?? '');
|
|
} elseif ($w !== null && $w < 100) {
|
|
$wobble[] = sprintf(' %-34s %.2f%% over 7 days', $d, $w);
|
|
}
|
|
}
|
|
if (!$down && !$wobble) exit(0);
|
|
echo "Uptime — 7 day review\n";
|
|
if ($down) { echo "\nCurrently down:\n"; foreach ($down as $l) echo "$l\n"; }
|
|
if ($wobble) { echo "\nNot perfect this week:\n"; foreach ($wobble as $l) echo "$l\n"; }
|
|
// Non-zero so the calling wrapper can notify on "there is something to say" without parsing.
|
|
exit(1);
|
|
}
|
|
|
|
// ── Status / events ───────────────────────────────────────────────────────────
|
|
if ($status || $events) {
|
|
$s = vv_uptime_read();
|
|
if (!$s) { echo "uptime.json is unreadable\n"; exit(0); }
|
|
$doms = $s['domains'] ?? [];
|
|
if ($events) {
|
|
$all = [];
|
|
foreach ($doms as $d => $r) foreach ($r['events'] ?? [] as $e) $all[] = $e + ['domain' => $d];
|
|
usort($all, fn($a, $b) => $b['ts'] <=> $a['ts']);
|
|
if (!$all) { echo "no state changes recorded\n"; exit(0); }
|
|
foreach (array_slice($all, 0, 40) as $e)
|
|
printf("%s %-34s %-5s %s\n", date('Y-m-d H:i', $e['ts']), $e['domain'],
|
|
strtoupper($e['to']), $e['detail'] ?? '');
|
|
exit(0);
|
|
}
|
|
printf("%-34s %-6s %8s %8s %8s %8s %s\n", 'domain', 'state', '24h', '7d', '30d', '1y', 'since');
|
|
foreach ($doms as $d => $r) {
|
|
$h24 = vv_uptime_window($r['hours'] ?? [], 24);
|
|
$d7 = vv_uptime_window($r['days'] ?? [], 7);
|
|
$d30 = vv_uptime_window($r['days'] ?? [], 30);
|
|
$y1 = vv_uptime_window($r['months'] ?? [], 12);
|
|
printf("%-34s %-6s %8s %8s %8s %8s %s\n", substr($d, 0, 34), $r['state'] ?? '-',
|
|
$h24 === null ? '-' : $h24 . '%', $d7 === null ? '-' : $d7 . '%',
|
|
$d30 === null ? '-' : $d30 . '%', $y1 === null ? '-' : $y1 . '%',
|
|
!empty($r['last_change']) ? date('m-d H:i', $r['last_change']) : '-');
|
|
}
|
|
printf("\n%d domains, last pass %s\n", count($doms),
|
|
!empty($s['last_pass']) ? date('Y-m-d H:i:s', $s['last_pass']) : 'never');
|
|
exit(0);
|
|
}
|
|
|
|
// Shared by --status, --report and the API, and defined once in include/auth.php so the three
|
|
// callers cannot drift apart on what "the last N" means. Buckets are keyed by time, so it is a key
|
|
// sort rather than an assumption about how many samples a period should contain — a pass that did
|
|
// not run leaves no bucket rather than a zero.
|
|
function vv_uptime_window(array $buckets, int $n): ?float {
|
|
return vv_auth_uptime_window($buckets, $n);
|
|
}
|
|
|
|
// ── One pass ──────────────────────────────────────────────────────────────────
|
|
$lock = @fopen(sys_get_temp_dir() . '/vv_uptime_probe.lock', 'c');
|
|
if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) { echo "another pass is running\n"; exit(0); }
|
|
|
|
try {
|
|
$v = vv_conf_vars();
|
|
if (strtolower(trim($v['UPTIME_PROBE_ENABLED'] ?? 'true')) === 'false') {
|
|
echo "UPTIME_PROBE_ENABLED is false\n"; exit(0);
|
|
}
|
|
$timeout = max(2, (int) ($v['UPTIME_PROBE_TIMEOUT'] ?? 8));
|
|
$listTtl = max(60, (int) ($v['UPTIME_PROBE_LIST_TTL'] ?? 300));
|
|
|
|
$store = vv_uptime_read();
|
|
if (!$store) { echo "uptime.json is malformed — refusing to overwrite it\n"; exit(1); }
|
|
|
|
// The domain list, refreshed occasionally rather than every minute. NPM's API is the source of
|
|
// truth for what is being served; asking it sixty times an hour for a list that changes weekly
|
|
// is the sort of thing that shows up later as unexplained load.
|
|
$now = time();
|
|
if (empty($store['list']) || ($now - ($store['list_at'] ?? 0)) > $listTtl) {
|
|
$p = vv_npm_list_proxies();
|
|
if ($p['ok'] ?? false) {
|
|
$list = [];
|
|
foreach ($p['proxies'] as $h) {
|
|
// A disabled host is not expected to answer, so probing it would record a
|
|
// permanent outage for something switched off on purpose.
|
|
if (($h['enabled'] ?? true) === false) continue;
|
|
foreach ($h['domain_names'] ?? [] as $d) {
|
|
$d = strtolower(trim((string) $d));
|
|
// A wildcard is not a hostname you can connect to.
|
|
if ($d !== '' && !str_contains($d, '*')) $list[$d] = true;
|
|
}
|
|
}
|
|
if ($list) { $store['list'] = array_keys($list); $store['list_at'] = $now; }
|
|
}
|
|
}
|
|
$domains = $store['list'] ?? [];
|
|
if (!$domains) { echo "no domains to probe\n"; exit(0); }
|
|
|
|
// ── Probe, in parallel ──
|
|
$mh = curl_multi_init();
|
|
$handles = [];
|
|
foreach ($domains as $d) {
|
|
$ch = curl_init('https://' . $d . '/');
|
|
curl_setopt_array($ch, [
|
|
CURLOPT_NOBODY => true, // HEAD — liveness, not content
|
|
CURLOPT_FOLLOWLOCATION => false, // a redirect to the auth portal is the answer
|
|
CURLOPT_TIMEOUT => $timeout,
|
|
CURLOPT_CONNECTTIMEOUT => min($timeout, 5),
|
|
CURLOPT_USERAGENT => VV_UPTIME_UA,
|
|
CURLOPT_RETURNTRANSFER => true,
|
|
// These are this host's own certificates behind its own proxy. Verification is left on
|
|
// because a cert that stopped validating is exactly the failure worth catching.
|
|
CURLOPT_SSL_VERIFYPEER => true,
|
|
CURLOPT_SSL_VERIFYHOST => 2,
|
|
]);
|
|
curl_multi_add_handle($mh, $ch);
|
|
$handles[$d] = $ch;
|
|
}
|
|
$t0 = microtime(true);
|
|
$running = null;
|
|
do {
|
|
curl_multi_exec($mh, $running);
|
|
if ($running) curl_multi_select($mh, 1.0);
|
|
} while ($running);
|
|
|
|
$upN = $downN = 0;
|
|
$downList = [];
|
|
$changes = [];
|
|
$doms = $store['domains'] ?? [];
|
|
$hourKey = date('YmdH', $now);
|
|
$dayKey = date('Ymd', $now);
|
|
$monKey = date('Ym', $now);
|
|
|
|
foreach ($handles as $d => $ch) {
|
|
$errno = curl_errno($ch);
|
|
$code = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
|
$ms = (int) round(curl_getinfo($ch, CURLINFO_TOTAL_TIME) * 1000);
|
|
$err = curl_error($ch);
|
|
curl_multi_remove_handle($mh, $ch);
|
|
curl_close($ch);
|
|
|
|
$up = ($errno === 0 && $code > 0 && $code < 500);
|
|
$detail = $errno !== 0 ? ($err ?: 'connection failed') : ('HTTP ' . $code);
|
|
$up ? $upN++ : $downN++;
|
|
if (!$up) $downList[] = "$d — $detail";
|
|
|
|
$r = $doms[$d] ?? ['checks' => 0, 'up' => 0, 'down' => 0, 'state' => null,
|
|
'last_change' => null, 'samples' => [], 'hours' => [], 'days' => [],
|
|
'months' => [], 'events' => [], 'since' => $now];
|
|
$r['checks']++;
|
|
$up ? $r['up']++ : $r['down']++;
|
|
$r['last_code'] = $code;
|
|
$r['last_ms'] = $ms;
|
|
$r['last_at'] = $now;
|
|
$r['last_detail'] = $detail;
|
|
|
|
$newState = $up ? 'up' : 'down';
|
|
if (($r['state'] ?? null) !== $newState) {
|
|
// The first observation is not a transition — there was no previous state to leave.
|
|
if ($r['state'] !== null) {
|
|
$ev = ['ts' => $now, 'to' => $newState, 'detail' => $detail];
|
|
$r['events'][] = $ev;
|
|
$r['events'] = array_slice($r['events'], -VV_EVENTS_KEEP);
|
|
$changes[] = "$d → " . strtoupper($newState) . " ($detail)";
|
|
}
|
|
$r['state'] = $newState;
|
|
$r['last_change'] = $now;
|
|
}
|
|
|
|
$r['samples'][] = $up ? 1 : 0;
|
|
if (count($r['samples']) > VV_SAMPLES_KEEP)
|
|
$r['samples'] = array_slice($r['samples'], -VV_SAMPLES_KEEP);
|
|
|
|
foreach ([['hours', $hourKey, VV_HOURS_KEEP],
|
|
['days', $dayKey, VV_DAYS_KEEP],
|
|
['months', $monKey, VV_MONTHS_KEEP]] as [$k, $key, $keep]) {
|
|
$b = $r[$k][$key] ?? ['u' => 0, 't' => 0];
|
|
$b['t']++;
|
|
if ($up) $b['u']++;
|
|
$r[$k][$key] = $b;
|
|
if (count($r[$k]) > $keep) { krsort($r[$k]); $r[$k] = array_slice($r[$k], 0, $keep, true); ksort($r[$k]); }
|
|
}
|
|
|
|
$doms[$d] = $r;
|
|
}
|
|
curl_multi_close($mh);
|
|
|
|
// Retire domains nothing has probed in a long time. Keyed on last_at rather than on absence
|
|
// from the current list, which matters when NPM is unreachable: the cached list keeps being
|
|
// probed, every domain keeps getting a last_at, and an NPM outage therefore cannot empty the
|
|
// store. Only a domain that genuinely left the list stops being stamped.
|
|
$cutoff = $now - (VV_DOMAIN_DROP_DAYS * 86400);
|
|
$dropped = [];
|
|
foreach ($doms as $d => $r) {
|
|
// A record with no last_at at all is kept. It should not be possible — every probe stamps
|
|
// it — and deleting on missing data is the wrong way round for something irreversible.
|
|
if (isset($r['last_at']) && $r['last_at'] < $cutoff) { unset($doms[$d]); $dropped[] = $d; }
|
|
}
|
|
|
|
$store['domains'] = $doms;
|
|
$store['last_pass'] = $now;
|
|
|
|
printf("%d domains — %d up, %d down, %.1fs%s\n", count($handles), $upN, $downN,
|
|
microtime(true) - $t0, $changes ? '' : ' (no state changes)');
|
|
foreach ($changes as $c) echo " $c\n";
|
|
foreach ($downList as $c) echo " DOWN $c\n";
|
|
// Named, not silent. Dropping a record throws away months of history, and a line in the log is
|
|
// the only trace that it was this and not the store being reset by something else.
|
|
foreach ($dropped as $d)
|
|
echo " RETIRED $d — not probed in " . VV_DOMAIN_DROP_DAYS . " days\n";
|
|
|
|
if ($dryRun) { echo "dry run — nothing written\n"; exit(0); }
|
|
if (!vv_uptime_write($store)) { echo 'could not write ' . vv_uptime_path() . "\n"; exit(1); }
|
|
exit(0);
|
|
|
|
} finally {
|
|
flock($lock, LOCK_UN);
|
|
fclose($lock);
|
|
}
|