Gmer4Lfe
99b58c0c4f
A low uptime figure, a refused login and a certificate that stopped renewing all looked the same from the row: a number, with the reason split across NPM, an Authelia config and the directory. The why-check goes and looks — TCP to the forward target, HTTP through the proxy, a second handshake with verification off to tell a broken certificate from a broken service. Forward hosts are docker names that only resolve on NPM's network, so an unresolvable one is redirected to the container address and the substitution is reported; a check that could not be made must never read as a check that failed. The access simulator walks the rules the way Authelia does and shows the ones it stepped over, reading whichever instance the chosen host points at rather than the one conf names — there are two here. Cert triage counts runs rather than log lines and orders by rotation suffix rather than mtime, both of which change the answer.
305 lines
15 KiB
PHP
305 lines
15 KiB
PHP
<?php
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
// PURPOSE
|
|
// Probes every hostname NPM serves, from outside the proxy, and records whether it answered.
|
|
// Produces the uptime percentages and the recent-history strip on the Proxies tab, and the list
|
|
// of anything that had a wobble for the Sunday report.
|
|
//
|
|
// WHY IT EXISTS
|
|
// NPM's access log says what happened when somebody visited. It says nothing at all about a host
|
|
// nobody visited, which is exactly the host most likely to be quietly broken. Five hosts here
|
|
// have been returning errors on every request for months without anything noticing.
|
|
//
|
|
// OPERATIONAL MODEL
|
|
// One pass = one sample per domain, all issued in parallel through curl_multi so the wall time
|
|
// is roughly the slowest domain rather than the sum of all of them. Intended to run every
|
|
// minute; it is safe at any interval because everything is stored as counts and buckets rather
|
|
// than assuming a cadence.
|
|
//
|
|
// up = the host answered with an HTTP status below 500.
|
|
// down = nothing answered, the TLS handshake failed, or it answered 5xx.
|
|
//
|
|
// A 401 or a 302 to the Authelia portal is UP. The question this asks is "is the server there",
|
|
// and an auth redirect is the strongest possible evidence that it is. Counting a protected site
|
|
// as down would mark every guarded host on this installation permanently offline.
|
|
//
|
|
// DESIGN PRINCIPLES
|
|
// Probes carry a User-Agent that the access-log aggregator ignores.
|
|
// Thirty-five domains once a minute is fifty thousand requests a day landing in the very
|
|
// logs Tools/npm_access_stats.sh counts. Left unmarked, this monitor would become the
|
|
// majority of the traffic it reports and bury whatever real use these hosts get.
|
|
//
|
|
// Bounded storage, decided up front.
|
|
// Per domain: lifetime counters, the last 60 samples for the strip, 48 hourly buckets and
|
|
// 30 daily ones. That is a fixed size — a per-minute probe kept as raw samples would be
|
|
// fifty thousand rows a day and the file would be the problem instead of the answer.
|
|
//
|
|
// The domain list follows NPM, not a hand-kept list in conf.
|
|
// A host added in the Proxies tab starts being probed without anyone remembering to add it
|
|
// somewhere else. The list is cached so this does not call the NPM API every minute.
|
|
//
|
|
// OPERATIONAL SAFEGUARDS
|
|
// Non-fatal, always: no NPM, no credentials, no domains — exits 0.
|
|
// One pass at a time, flock non-blocking, so a slow pass cannot overlap the next minute's.
|
|
// Store written tmp + rename, verified before it replaces the previous file.
|
|
// HEAD, not GET, and nothing is followed — this is a liveness check, not a crawler.
|
|
//
|
|
// RUNTIME MODES
|
|
// uptime_probe.php one pass
|
|
// uptime_probe.php --dry-run probe and report, write nothing
|
|
// uptime_probe.php --status per-domain uptime table
|
|
// uptime_probe.php --events recent state changes, newest first
|
|
//
|
|
// CONFIGURATION
|
|
// UPTIME_PROBE_ENABLED master switch (default true)
|
|
// UPTIME_PROBE_TIMEOUT seconds per domain (default 8)
|
|
// UPTIME_PROBE_LIST_TTL seconds to reuse the cached domain list (default 900)
|
|
// VV_UPTIME_UA the User-Agent, matched by npm_access_stats.php
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
require_once dirname(__DIR__) . '/include/auth.php';
|
|
|
|
// Shared with npm_access_stats.php, which drops any log line containing it. Changing this in one
|
|
// place and not the other turns the monitor's own traffic back into counted requests.
|
|
const VV_UPTIME_UA = 'Varaverk-Uptime/1.0';
|
|
const VV_SAMPLES_KEEP = 60; // one hour at a one-minute cadence
|
|
const VV_HOURS_KEEP = 48;
|
|
const VV_DAYS_KEEP = 30;
|
|
const VV_EVENTS_KEEP = 20;
|
|
|
|
$dryRun = in_array('--dry-run', $argv, true);
|
|
$status = in_array('--status', $argv, true);
|
|
$events = in_array('--events', $argv, true);
|
|
$report = in_array('--report', $argv, true);
|
|
|
|
function vv_uptime_path(): string {
|
|
return rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/uptime.json';
|
|
}
|
|
|
|
function vv_uptime_read(): array {
|
|
$p = vv_uptime_path();
|
|
if (!is_file($p)) return ['domains' => []];
|
|
$j = json_decode((string) @file_get_contents($p), true);
|
|
if (!is_array($j) || !isset($j['domains']) || !is_array($j['domains'])) return [];
|
|
return $j;
|
|
}
|
|
|
|
function vv_uptime_write(array $d): bool {
|
|
$p = vv_uptime_path();
|
|
if (!is_dir(dirname($p)) && !@mkdir(dirname($p), 0755, true)) return false;
|
|
$d['updated'] = time();
|
|
$json = json_encode($d, JSON_UNESCAPED_SLASHES);
|
|
if ($json === false) return false;
|
|
$tmp = $p . '.vv.tmp';
|
|
if (@file_put_contents($tmp, $json) === false) return false;
|
|
if (json_decode((string) @file_get_contents($tmp), true) === null) { @unlink($tmp); return false; }
|
|
if (!@rename($tmp, $p)) { @unlink($tmp); return false; }
|
|
return true;
|
|
}
|
|
|
|
// ── Report ────────────────────────────────────────────────────────────────────
|
|
// Anything that was not perfect over the last seven days, for the Sunday report. Prints nothing
|
|
// and exits 0 when every domain was clean — the orchestrator's job is to be quiet on a good week,
|
|
// and a report that always says something is a report nobody reads.
|
|
if ($report) {
|
|
$s = vv_uptime_read();
|
|
$doms = $s['domains'] ?? [];
|
|
if (!$doms) { exit(0); }
|
|
$down = $wobble = [];
|
|
foreach ($doms as $d => $r) {
|
|
$w = vv_uptime_window($r['days'] ?? [], 7);
|
|
if (($r['state'] ?? '') === 'down') {
|
|
$since = !empty($r['last_change']) ? ' since ' . date('D H:i', $r['last_change']) : '';
|
|
$down[] = sprintf(' %-34s DOWN%s — %s', $d, $since, $r['last_detail'] ?? '');
|
|
} elseif ($w !== null && $w < 100) {
|
|
$wobble[] = sprintf(' %-34s %.2f%% over 7 days', $d, $w);
|
|
}
|
|
}
|
|
if (!$down && !$wobble) exit(0);
|
|
echo "Uptime — 7 day review\n";
|
|
if ($down) { echo "\nCurrently down:\n"; foreach ($down as $l) echo "$l\n"; }
|
|
if ($wobble) { echo "\nNot perfect this week:\n"; foreach ($wobble as $l) echo "$l\n"; }
|
|
// Non-zero so the calling wrapper can notify on "there is something to say" without parsing.
|
|
exit(1);
|
|
}
|
|
|
|
// ── Status / events ───────────────────────────────────────────────────────────
|
|
if ($status || $events) {
|
|
$s = vv_uptime_read();
|
|
if (!$s) { echo "uptime.json is unreadable\n"; exit(0); }
|
|
$doms = $s['domains'] ?? [];
|
|
if ($events) {
|
|
$all = [];
|
|
foreach ($doms as $d => $r) foreach ($r['events'] ?? [] as $e) $all[] = $e + ['domain' => $d];
|
|
usort($all, fn($a, $b) => $b['ts'] <=> $a['ts']);
|
|
if (!$all) { echo "no state changes recorded\n"; exit(0); }
|
|
foreach (array_slice($all, 0, 40) as $e)
|
|
printf("%s %-34s %-5s %s\n", date('Y-m-d H:i', $e['ts']), $e['domain'],
|
|
strtoupper($e['to']), $e['detail'] ?? '');
|
|
exit(0);
|
|
}
|
|
printf("%-34s %-6s %8s %8s %8s %s\n", 'domain', 'state', '1h', '24h', '30d', 'since');
|
|
foreach ($doms as $d => $r) {
|
|
$h = vv_uptime_window($r['hours'] ?? [], 1);
|
|
$h24 = vv_uptime_window($r['hours'] ?? [], 24);
|
|
$d30 = vv_uptime_window($r['days'] ?? [], 30);
|
|
printf("%-34s %-6s %8s %8s %8s %s\n", substr($d, 0, 34), $r['state'] ?? '-',
|
|
$h === null ? '-' : $h . '%', $h24 === null ? '-' : $h24 . '%',
|
|
$d30 === null ? '-' : $d30 . '%',
|
|
!empty($r['last_change']) ? date('m-d H:i', $r['last_change']) : '-');
|
|
}
|
|
printf("\n%d domains, last pass %s\n", count($doms),
|
|
!empty($s['last_pass']) ? date('Y-m-d H:i:s', $s['last_pass']) : 'never');
|
|
exit(0);
|
|
}
|
|
|
|
// Shared by --status, --report and the API, and defined once in include/auth.php so the three
|
|
// callers cannot drift apart on what "the last N" means. Buckets are keyed by time, so it is a key
|
|
// sort rather than an assumption about how many samples a period should contain — a pass that did
|
|
// not run leaves no bucket rather than a zero.
|
|
function vv_uptime_window(array $buckets, int $n): ?float {
|
|
return vv_auth_uptime_window($buckets, $n);
|
|
}
|
|
|
|
// ── One pass ──────────────────────────────────────────────────────────────────
|
|
$lock = @fopen(sys_get_temp_dir() . '/vv_uptime_probe.lock', 'c');
|
|
if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) { echo "another pass is running\n"; exit(0); }
|
|
|
|
try {
|
|
$v = vv_conf_vars();
|
|
if (strtolower(trim($v['UPTIME_PROBE_ENABLED'] ?? 'true')) === 'false') {
|
|
echo "UPTIME_PROBE_ENABLED is false\n"; exit(0);
|
|
}
|
|
$timeout = max(2, (int) ($v['UPTIME_PROBE_TIMEOUT'] ?? 8));
|
|
$listTtl = max(60, (int) ($v['UPTIME_PROBE_LIST_TTL'] ?? 900));
|
|
|
|
$store = vv_uptime_read();
|
|
if (!$store) { echo "uptime.json is malformed — refusing to overwrite it\n"; exit(1); }
|
|
|
|
// The domain list, refreshed occasionally rather than every minute. NPM's API is the source of
|
|
// truth for what is being served; asking it sixty times an hour for a list that changes weekly
|
|
// is the sort of thing that shows up later as unexplained load.
|
|
$now = time();
|
|
if (empty($store['list']) || ($now - ($store['list_at'] ?? 0)) > $listTtl) {
|
|
$p = vv_npm_list_proxies();
|
|
if ($p['ok'] ?? false) {
|
|
$list = [];
|
|
foreach ($p['proxies'] as $h) {
|
|
// A disabled host is not expected to answer, so probing it would record a
|
|
// permanent outage for something switched off on purpose.
|
|
if (($h['enabled'] ?? true) === false) continue;
|
|
foreach ($h['domain_names'] ?? [] as $d) {
|
|
$d = strtolower(trim((string) $d));
|
|
// A wildcard is not a hostname you can connect to.
|
|
if ($d !== '' && !str_contains($d, '*')) $list[$d] = true;
|
|
}
|
|
}
|
|
if ($list) { $store['list'] = array_keys($list); $store['list_at'] = $now; }
|
|
}
|
|
}
|
|
$domains = $store['list'] ?? [];
|
|
if (!$domains) { echo "no domains to probe\n"; exit(0); }
|
|
|
|
// ── Probe, in parallel ──
|
|
$mh = curl_multi_init();
|
|
$handles = [];
|
|
foreach ($domains as $d) {
|
|
$ch = curl_init('https://' . $d . '/');
|
|
curl_setopt_array($ch, [
|
|
CURLOPT_NOBODY => true, // HEAD — liveness, not content
|
|
CURLOPT_FOLLOWLOCATION => false, // a redirect to the auth portal is the answer
|
|
CURLOPT_TIMEOUT => $timeout,
|
|
CURLOPT_CONNECTTIMEOUT => min($timeout, 5),
|
|
CURLOPT_USERAGENT => VV_UPTIME_UA,
|
|
CURLOPT_RETURNTRANSFER => true,
|
|
// These are this host's own certificates behind its own proxy. Verification is left on
|
|
// because a cert that stopped validating is exactly the failure worth catching.
|
|
CURLOPT_SSL_VERIFYPEER => true,
|
|
CURLOPT_SSL_VERIFYHOST => 2,
|
|
]);
|
|
curl_multi_add_handle($mh, $ch);
|
|
$handles[$d] = $ch;
|
|
}
|
|
$t0 = microtime(true);
|
|
$running = null;
|
|
do {
|
|
curl_multi_exec($mh, $running);
|
|
if ($running) curl_multi_select($mh, 1.0);
|
|
} while ($running);
|
|
|
|
$upN = $downN = 0;
|
|
$downList = [];
|
|
$changes = [];
|
|
$doms = $store['domains'] ?? [];
|
|
$hourKey = date('YmdH', $now);
|
|
$dayKey = date('Ymd', $now);
|
|
|
|
foreach ($handles as $d => $ch) {
|
|
$errno = curl_errno($ch);
|
|
$code = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
|
$ms = (int) round(curl_getinfo($ch, CURLINFO_TOTAL_TIME) * 1000);
|
|
$err = curl_error($ch);
|
|
curl_multi_remove_handle($mh, $ch);
|
|
curl_close($ch);
|
|
|
|
$up = ($errno === 0 && $code > 0 && $code < 500);
|
|
$detail = $errno !== 0 ? ($err ?: 'connection failed') : ('HTTP ' . $code);
|
|
$up ? $upN++ : $downN++;
|
|
if (!$up) $downList[] = "$d — $detail";
|
|
|
|
$r = $doms[$d] ?? ['checks' => 0, 'up' => 0, 'down' => 0, 'state' => null,
|
|
'last_change' => null, 'samples' => [], 'hours' => [], 'days' => [],
|
|
'events' => [], 'since' => $now];
|
|
$r['checks']++;
|
|
$up ? $r['up']++ : $r['down']++;
|
|
$r['last_code'] = $code;
|
|
$r['last_ms'] = $ms;
|
|
$r['last_at'] = $now;
|
|
$r['last_detail'] = $detail;
|
|
|
|
$newState = $up ? 'up' : 'down';
|
|
if (($r['state'] ?? null) !== $newState) {
|
|
// The first observation is not a transition — there was no previous state to leave.
|
|
if ($r['state'] !== null) {
|
|
$ev = ['ts' => $now, 'to' => $newState, 'detail' => $detail];
|
|
$r['events'][] = $ev;
|
|
$r['events'] = array_slice($r['events'], -VV_EVENTS_KEEP);
|
|
$changes[] = "$d → " . strtoupper($newState) . " ($detail)";
|
|
}
|
|
$r['state'] = $newState;
|
|
$r['last_change'] = $now;
|
|
}
|
|
|
|
$r['samples'][] = $up ? 1 : 0;
|
|
if (count($r['samples']) > VV_SAMPLES_KEEP)
|
|
$r['samples'] = array_slice($r['samples'], -VV_SAMPLES_KEEP);
|
|
|
|
foreach ([['hours', $hourKey, VV_HOURS_KEEP], ['days', $dayKey, VV_DAYS_KEEP]] as [$k, $key, $keep]) {
|
|
$b = $r[$k][$key] ?? ['u' => 0, 't' => 0];
|
|
$b['t']++;
|
|
if ($up) $b['u']++;
|
|
$r[$k][$key] = $b;
|
|
if (count($r[$k]) > $keep) { krsort($r[$k]); $r[$k] = array_slice($r[$k], 0, $keep, true); ksort($r[$k]); }
|
|
}
|
|
|
|
$doms[$d] = $r;
|
|
}
|
|
curl_multi_close($mh);
|
|
|
|
$store['domains'] = $doms;
|
|
$store['last_pass'] = $now;
|
|
|
|
printf("%d domains — %d up, %d down, %.1fs%s\n", count($handles), $upN, $downN,
|
|
microtime(true) - $t0, $changes ? '' : ' (no state changes)');
|
|
foreach ($changes as $c) echo " $c\n";
|
|
foreach ($downList as $c) echo " DOWN $c\n";
|
|
|
|
if ($dryRun) { echo "dry run — nothing written\n"; exit(0); }
|
|
if (!vv_uptime_write($store)) { echo 'could not write ' . vv_uptime_path() . "\n"; exit(1); }
|
|
exit(0);
|
|
|
|
} finally {
|
|
flock($lock, LOCK_UN);
|
|
fclose($lock);
|
|
}
|