diff --git a/Plugin/unraid/Tools/cert_triage.php b/Plugin/unraid/Tools/cert_triage.php new file mode 100755 index 0000000..93e0619 --- /dev/null +++ b/Plugin/unraid/Tools/cert_triage.php @@ -0,0 +1,78 @@ + 6) printf(" … and %d more\n", count($c['domains']) - 6); +} + +if ($r['reading']) { echo "\n"; foreach ($r['reading'] as $l) echo " $l\n"; } +exit(0); diff --git a/Plugin/unraid/Tools/cert_triage.sh b/Plugin/unraid/Tools/cert_triage.sh new file mode 100644 index 0000000..d964267 --- /dev/null +++ b/Plugin/unraid/Tools/cert_triage.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# ============================================================================================== +# ==================================== Cert Triage ============================================= +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Reads certbot's own logs and names why renewals failed, in the handful of categories they +# actually fall into. +# +# cert_history.sh counts failures — it notices an expiry in the past. It cannot say why. The why +# is in certbot's log, which here is 1001 rotated files and 639 MB, and the last time anyone +# answered "why did ten certificates stop renewing" they read them by hand. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# A wrapper. The work is in cert_triage.php. +# +# Counts runs, not lines. One log file is one certbot invocation, and one failure writes its +# reason three times — in the ACME response, the traceback, and certbot's own summary. Counting +# lines reports a single failure as three and inflates whichever category is most verbose. +# +# Reads the newest logs by rotation suffix, never by mtime. Every file here carries the same +# mtime because they are synced as a set, so mtime order is meaningless. +# +# The categories are separated into causes and consequences. Rate limiting is nearly always +# downstream — retries against a hostname with no DNS record exhaust the allowance, which then +# fails renewals for domains that have nothing wrong with them. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# cert_triage.sh summary — categories, affected domains, and the causal reading +# cert_triage.sh --json the same as JSON, for the Certs tab +# cert_triage.sh --files=N override how many rotated logs to read +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# CERT_TRIAGE_FILES rotated logs to read, newest first +# CERT_TRIAGE_MAX_BYTES bytes read from the end of each +# CERT_TRIAGE_LOG_DIR override the log directory; normally found from the NPM container +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +php "$SCRIPT_DIR/cert_triage.php" "$@" diff --git a/Plugin/unraid/Tools/uptime_probe.php b/Plugin/unraid/Tools/uptime_probe.php index eeb9e92..8e462d7 100644 --- a/Plugin/unraid/Tools/uptime_probe.php +++ b/Plugin/unraid/Tools/uptime_probe.php @@ -97,10 +97,6 @@ function vv_uptime_write(array $d): bool { return true; } -function vv_uptime_pct(int $up, int $total): ?float { - return $total > 0 ? round($up / $total * 100, 2) : null; -} - // ── Report ──────────────────────────────────────────────────────────────────── // Anything that was not perfect over the last seven days, for the Sunday report. Prints nothing // and exits 0 when every domain was clean — the orchestrator's job is to be quiet on a good week, @@ -157,14 +153,12 @@ if ($status || $events) { exit(0); } -// Shared by --status, --report and the API. Buckets are keyed by time so "the last N" is a key sort, not an assumption about how many -// samples a period should contain — a pass that did not run leaves no bucket rather than a zero. +// Shared by --status, --report and the API, and defined once in include/auth.php so the three +// callers cannot drift apart on what "the last N" means. Buckets are keyed by time, so it is a key +// sort rather than an assumption about how many samples a period should contain — a pass that did +// not run leaves no bucket rather than a zero. function vv_uptime_window(array $buckets, int $n): ?float { - if (!$buckets) return null; - krsort($buckets); - $u = $t = 0; - foreach (array_slice($buckets, 0, $n, true) as $b) { $u += $b['u'] ?? 0; $t += $b['t'] ?? 0; } - return vv_uptime_pct($u, $t); + return vv_auth_uptime_window($buckets, $n); } // ── One pass ────────────────────────────────────────────────────────────────── diff --git a/Plugin/unraid/api/auth.php b/Plugin/unraid/api/auth.php index b050c26..171f1c1 100644 --- a/Plugin/unraid/api/auth.php +++ b/Plugin/unraid/api/auth.php @@ -66,7 +66,10 @@ // actions are reads and are deliberately outside it. See README-unraid.md. // // REQUEST -// GET ?action=npm_proxies | npm_certs | lldap_users | lldap_groups | authelia_rules +// GET ?action=npm_proxies | npm_certs | npm_stats | npm_uptime | lldap_users | lldap_groups +// | authelia_rules +// GET ?action=npm_why id +// GET ?action=access_check domain, uid, path // POST action=npm_create data= // POST action=npm_update id, data= // POST action=npm_delete id @@ -103,8 +106,9 @@ require_once dirname(__DIR__) . '/include/auth.php'; const VV_AUTH_ACTION_PANEL = [ // GET 'npm_proxies' => 'proxies', 'npm_certs' => 'proxies', 'npm_stats' => 'proxies', 'npm_uptime' => 'proxies', + 'npm_why' => 'proxies', 'lldap_users' => 'users', 'lldap_groups' => 'users', 'lldap_avatar' => 'users', - 'authelia_rules' => 'acl', + 'authelia_rules' => 'acl', 'access_check' => 'acl', // POST 'npm_create' => 'proxies', 'npm_update' => 'proxies', 'npm_delete' => 'proxies', 'npm_toggle' => 'proxies', @@ -115,14 +119,11 @@ const VV_AUTH_ACTION_PANEL = [ 'authelia_save' => 'acl', ]; -// Same windowing rule as uptime_probe.php: buckets are keyed by time, so "the last N" is a key -// sort rather than an assumption that every period produced a sample. +// The windowing rule lives in include/auth.php and is shared with Tools/uptime_probe.php. It was +// three separate copies of the same six lines, which is three places for the definition of "the +// last 24 hours" to drift apart while every one of them keeps returning a plausible number. function vv_uptime_window_api(array $buckets, int $n): ?float { - if (!$buckets) return null; - krsort($buckets); - $u = $t = 0; - foreach (array_slice($buckets, 0, $n, true) as $b) { $u += $b['u'] ?? 0; $t += $b['t'] ?? 0; } - return $t > 0 ? round($u / $t * 100, 2) : null; + return vv_auth_uptime_window($buckets, $n); } function vv_auth_action_allowed(string $action): bool { @@ -195,6 +196,28 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') { exit; } + // Why one host is not at 100%. The only read here that goes and looks rather than serving a + // stored figure: it opens a socket to the forward target, asks the domain itself, and inspects + // the containers. Slow by the standards of this file — several seconds — which is why it is one + // host on demand and never part of the list load. + // + // A read, so it stays in the GET arm with the other reads. It is worth being explicit that this + // is safe to leave outside the CSRF guard: every call it makes is a GET, a HEAD, a TCP connect + // or a file read, so the worst a forged request achieves is making this machine look at itself. + if ($action === 'npm_why') { + echo json_encode(vv_npm_why((int) ($_GET['id'] ?? 0))); + exit; + } + + // Can this user open this URL, and what decided it. Reads NPM, the Authelia instance that this + // particular host talks to, and the directory — the three places the answer is split across. + if ($action === 'access_check') { + echo json_encode(vv_auth_access_check((string) ($_GET['domain'] ?? ''), + (string) ($_GET['uid'] ?? ''), + (string) ($_GET['path'] ?? '/'))); + exit; + } + $result = match ($action) { 'npm_proxies' => vv_npm_list_proxies(), 'npm_certs' => ['ok' => true, 'certs' => vv_npm_list_certs()], diff --git a/Plugin/unraid/api/cert.php b/Plugin/unraid/api/cert.php index d796a5c..38ff031 100644 --- a/Plugin/unraid/api/cert.php +++ b/Plugin/unraid/api/cert.php @@ -308,5 +308,14 @@ if (!file_exists($cacheFile)) { exit; } +// ── Why renewals failed ─────────────────────────────────────────────────────── +// Reads certbot's own logs and names the categories the failures fall into. A read, so GET: it +// opens files and nothing else. Bounded inside vv_cert_triage() by file count and bytes per file, +// because the log directory here is 639 MB and a page request must not depend on its size. +if ($action === 'triage') { + echo json_encode(vv_cert_triage()); + exit; +} + $data = json_decode(file_get_contents($cacheFile), true) ?: []; echo json_encode(array_merge(['ok' => true], $data)); diff --git a/Plugin/unraid/include/auth.php b/Plugin/unraid/include/auth.php index 92cda7c..9c6cdf4 100644 --- a/Plugin/unraid/include/auth.php +++ b/Plugin/unraid/include/auth.php @@ -61,6 +61,9 @@ // vv_auth_last_transport() // NPM vv_npm_list_proxies(), vv_npm_list_certs(), vv_npm_create_proxy(), // vv_npm_update_proxy(), vv_npm_delete_proxy(), vv_npm_toggle_proxy() +// Diagnosis vv_npm_why(), vv_npm_why_findings(), vv_auth_uptime_window(), vv_auth_db_file(), +// vv_auth_tcp_probe(), vv_auth_http_probe(), vv_auth_container_for() +// — read-only. The only group here that changes nothing. // LLDAP vv_lldap_list_users(), vv_lldap_list_groups(), vv_lldap_create_user(), // vv_lldap_update_user(), vv_lldap_delete_user(), vv_lldap_set_password(), // vv_lldap_create_group(), vv_lldap_delete_group(), @@ -315,6 +318,398 @@ function vv_npm_toggle_proxy(int $id, bool $enabled): array { return ['ok' => true]; } +// ── Why is this host not at 100% ────────────────────────────────────────────── + +// The one implementation of the uptime windowing rule. Buckets are keyed by time, so "the last N" +// is a key sort rather than an assumption that every period produced a sample — a pass that did not +// run leaves no bucket at all rather than a zero, and averaging over a count would read a probe +// outage as a service outage. Tools/uptime_probe.php and api/auth.php both defer to this. +function vv_auth_uptime_window(array $buckets, int $n): ?float { + if (!$buckets) return null; + krsort($buckets); + $u = $t = 0; + foreach (array_slice($buckets, 0, $n, true) as $b) { $u += $b['u'] ?? 0; $t += $b['t'] ?? 0; } + return $t > 0 ? round($u / $t * 100, 2) : null; +} + +function vv_auth_db_file(string $name): string { + return rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/' . $name; +} + +// One TCP connect, timed. The cheapest question that separates "the application is broken" from +// "nothing is there at all", and the one the proxy itself cannot answer — NPM reports a 502 for a +// refused connection, a closed port and a hung process alike. +function vv_auth_tcp_probe(string $host, int $port, int $timeout = 4): array { + if ($host === '' || $port <= 0) return ['ok' => false, 'err' => 'no forward target configured']; + $t0 = microtime(true); + $errno = 0; $errstr = ''; + // @ because a refused connection and an unresolvable name are both expected answers here, and + // a warning raised into the JSON body would corrupt the response this is reported in. + $fp = @fsockopen($host, $port, $errno, $errstr, $timeout); + $ms = (int) round((microtime(true) - $t0) * 1000); + if ($fp === false) return ['ok' => false, 'ms' => $ms, 'err' => $errstr ?: ('errno ' . $errno)]; + fclose($fp); + return ['ok' => true, 'ms' => $ms]; +} + +// A HEAD against a URL, reporting the same three things for every probe so the caller can compare +// the front door and the back door without special-casing either. +// +// $verify is the whole point of the second call this makes: an identical request that succeeds only +// with verification off says the certificate is the fault and the service behind it is fine, which +// is otherwise indistinguishable from the site being down. +function vv_auth_http_probe(string $url, bool $verify, int $timeout = 5): array { + $ch = curl_init($url); + curl_setopt_array($ch, [ + CURLOPT_NOBODY => true, + CURLOPT_FOLLOWLOCATION => false, // a redirect to the auth portal is an answer, not a step + CURLOPT_TIMEOUT => $timeout, + CURLOPT_CONNECTTIMEOUT => min($timeout, 4), + // Marked as the monitor so this cannot land in the access log as a real request and inflate + // the very traffic figures shown beside it. Same string npm_access_stats.php drops. + CURLOPT_USERAGENT => 'Varaverk-Uptime/1.0', + CURLOPT_RETURNTRANSFER => true, + CURLOPT_SSL_VERIFYPEER => $verify, + CURLOPT_SSL_VERIFYHOST => $verify ? 2 : 0, + ]); + curl_exec($ch); + $r = ['code' => (int) curl_getinfo($ch, CURLINFO_HTTP_CODE), + 'ms' => (int) round(curl_getinfo($ch, CURLINFO_TOTAL_TIME) * 1000), + 'err' => curl_error($ch)]; + curl_close($ch); + return $r; +} + +// Which container is behind a forward target, if any. Matched on the name first and the container +// IP second, because both forms are in use here — some hosts forward to a container name on a +// custom network and some to an address on br0. +// +// Returns null rather than guessing. A forward target that is another machine entirely is a normal +// configuration, and reporting "no container" for it is correct, not a failure to find one. +function vv_auth_container_for(string $fwdHost): ?array { + if ($fwdHost === '') return null; + require_once __DIR__ . '/docker.php'; + $all = vv_dk_inspect_all(); + if (!$all) return null; + + $hit = function (string $name, array $c, string $how): array { + $nets = $c['networks'] ?? []; + return ['name' => $name, 'running' => $c['running'], 'status' => $c['status'], 'match' => $how, + 'networks' => $nets, 'ip' => $nets ? reset($nets) : '', + 'network' => $nets ? (string) array_key_first($nets) : '']; + }; + + $needle = strtolower($fwdHost); + foreach ($all as $name => $c) if (strtolower($name) === $needle) return $hit($name, $c, 'name'); + foreach ($all as $name => $c) + foreach ($c['networks'] ?? [] as $ip) if ($ip === $fwdHost) return $hit($name, $c, 'ip'); + return null; +} + +// Where this process should actually knock, which is not always what the proxy host says. +// +// Most forward targets here are container names on a user-defined docker network. Those names are +// resolved by docker's embedded DNS, which only the containers on that network can see — NPM +// resolves NextCloud perfectly and PHP running on the host cannot resolve it at all. A check that +// treated its own resolution failure as evidence would report every one of them as dead, which is +// the exact false alarm this whole dialog exists to stop someone chasing. +// +// So an unresolvable name that matches a running container is redirected to that container's +// address, and the substitution is reported rather than hidden — the reader needs to know which +// address the result below actually describes. +function vv_auth_probe_target(string $fwdHost, ?array $container): array { + if ($fwdHost === '' || filter_var($fwdHost, FILTER_VALIDATE_IP)) return ['host' => $fwdHost, 'note' => '']; + // gethostbyname() hands back its input unchanged when it cannot resolve — the documented way it + // fails, and the reason this is a comparison rather than a truthiness test. + if (@gethostbyname($fwdHost) !== $fwdHost) return ['host' => $fwdHost, 'note' => '']; + + if ($container && ($container['ip'] ?? '') !== '') + return ['host' => $container['ip'], + 'note' => $fwdHost . ' is a docker name that only resolves on the ' . ($container['network'] ?: 'proxy') + . ' network, so this checked the container address ' . $container['ip'] . ' instead.']; + + return ['host' => $fwdHost, + 'note' => $fwdHost . ' does not resolve from this host and matches no container here, so the direct ' + . 'check below could not be made. What the proxy itself reports is the reliable part.']; +} + +// Everything known about why one proxy host is not at 100%, gathered in one pass. +// +// The Proxies tab shows an uptime percentage and nothing about what is behind it, and the causes +// look identical from the row: the application is down, its container is not running, the +// certificate stopped validating, or the proxy reaches the application perfectly and the +// application is the thing returning 5xx. Those are four different jobs and the row cannot tell +// them apart, so every low figure has so far meant opening NPM, then Docker, then a terminal. +// +// Three of the four sources are already on disk — the probe history, the access-log totals, NPM's +// own record. The fourth is the part no stored figure can answer: what happens right now, asked +// separately of the front door and the back. A host that answers on 10.0.0.5:8096 but fails through +// https://name/ is a proxy or certificate fault; one that fails both is the service itself. +// +// Read-only by construction. Every call below is a GET, a HEAD, a TCP connect or a file read — +// nothing here restarts, rewrites or retries anything, because the value of a diagnosis is that it +// can be run on a host that is limping without being the thing that finishes it off. +function vv_npm_why(int $id): array { + $p = vv_npm_list_proxies(); + if (!($p['ok'] ?? false)) return ['ok' => false, 'error' => $p['error'] ?? 'Could not read proxy hosts from NPM']; + + $host = null; + foreach ($p['proxies'] as $h) if ((int) ($h['id'] ?? 0) === $id) { $host = $h; break; } + if (!$host) return ['ok' => false, 'error' => "No proxy host with id $id — the list may be stale, reload the tab."]; + + $adv = trim((string) ($host['advanced_config'] ?? '')); + $guarded = str_contains($adv, 'auth_request'); + $fwdHost = trim((string) ($host['forward_host'] ?? '')); + $fwdPort = (int) ($host['forward_port'] ?? 0); + $fwdScheme = (string) ($host['forward_scheme'] ?? 'http'); + $enabled = ($host['enabled'] ?? true) ? true : false; + + // Wildcards are excluded for the same reason the probe excludes them: *.example.com is not a + // hostname anything can connect to, so a live check against it would report a fault that only + // describes the check. + $domains = []; + foreach ($host['domain_names'] ?? [] as $d) { + $d = strtolower(trim((string) $d)); + if ($d !== '' && !str_contains($d, '*')) $domains[] = $d; + } + + $out = [ + 'ok' => true, + 'id' => $id, + 'enabled' => $enabled, + 'domains' => $host['domain_names'] ?? [], + 'forward' => $fwdScheme . '://' . $fwdHost . ':' . $fwdPort, + 'guarded' => $guarded, + 'checked' => time(), + ]; + + // ── Certificate ── + // Expiry is the single most common reason a host that worked for months stops, and the row + // cannot show it because the row is about uptime. + $cert = null; + $certId = (int) ($host['certificate_id'] ?? 0); + if ($certId > 0) { + foreach (vv_npm_list_certs() as $c) { + if ((int) ($c['id'] ?? 0) !== $certId) continue; + $exp = strtotime((string) ($c['expires_on'] ?? '')) ?: null; + $cert = [ + 'name' => $c['nice_name'] ?? implode(', ', $c['domain_names'] ?? []), + 'provider' => $c['provider'] ?? '', + 'expires' => $exp, + 'days_left' => $exp ? (int) floor(($exp - time()) / 86400) : null, + ]; + break; + } + } + $out['cert'] = $cert; + + // ── Recorded history ── + $u = is_file(vv_auth_db_file('uptime.json')) + ? (json_decode((string) @file_get_contents(vv_auth_db_file('uptime.json')), true) ?: []) : []; + $recs = []; + foreach ($domains as $d) { + $r = $u['domains'][$d] ?? null; + if (!$r) continue; + // The hours that actually lost something, rather than all 48. "Every hour lost two samples" + // and "one hour lost forty" are the same daily percentage and completely different faults, + // and this is the only place that distinction survives. + $bad = []; + $hours = $r['hours'] ?? []; + krsort($hours); + foreach (array_slice($hours, 0, 24, true) as $k => $b) { + $t = $b['t'] ?? 0; $up = $b['u'] ?? 0; + if ($t > 0 && $up < $t) $bad[] = ['hour' => $k, 'up' => $up, 'total' => $t]; + } + $recs[$d] = [ + 'state' => $r['state'] ?? null, + 'h1' => vv_auth_uptime_window($r['hours'] ?? [], 1), + 'h24' => vv_auth_uptime_window($r['hours'] ?? [], 24), + 'd30' => vv_auth_uptime_window($r['days'] ?? [], 30), + 'last_code' => $r['last_code'] ?? null, + 'last_detail' => $r['last_detail'] ?? null, + 'last_change' => $r['last_change'] ?? null, + 'last_ms' => $r['last_ms'] ?? null, + 'checks' => $r['checks'] ?? 0, + // Newest first — a flap is read backwards from now, not forwards from whenever the + // record happens to start. + 'events' => array_slice(array_reverse($r['events'] ?? []), 0, 8), + 'bad_hours' => $bad, + ]; + } + $out['history'] = $recs; + + // ── Access-log totals ── + $a = is_file(vv_auth_db_file('npm_access.json')) + ? (json_decode((string) @file_get_contents(vv_auth_db_file('npm_access.json')), true) ?: []) : []; + $out['traffic'] = $a['hosts'][(string) $id] ?? null; + + // ── Live, right now ── + $cont = vv_auth_container_for($fwdHost); + $target = vv_auth_probe_target($fwdHost, $cont); + $out['upstream'] = [ + 'host' => $fwdHost, + 'port' => $fwdPort, + 'probed' => $target['host'], + 'note' => $target['note'], + 'container' => $cont, + 'tcp' => vv_auth_tcp_probe($target['host'], $fwdPort), + ]; + // Only worth asking once something is listening — an HTTP probe of a closed port re-reports the + // TCP failure in a less specific form. + if ($out['upstream']['tcp']['ok'] ?? false) { + // Verification off deliberately: this is an internal hop to an address on this machine's own + // network, usually plain HTTP and usually a self-signed certificate when it is not. The + // question here is whether the application answers, and the certificate question is asked + // at the front door where it actually applies. + $out['upstream']['http'] = vv_auth_http_probe($fwdScheme . '://' . $target['host'] . ':' . $fwdPort . '/', false); + } + + // Bounded. A host carrying a dozen names would otherwise turn one button press into a dozen + // sequential TLS handshakes, and the first few answer the question. + $out['live'] = []; + foreach (array_slice($domains, 0, 4) as $d) { + $r = vv_auth_http_probe('https://' . $d . '/', true); + // The second call is the diagnosis, not a retry: succeeding here after failing above is + // what proves the certificate rather than the service. + if ($r['err'] !== '' && preg_match('/certificat|SSL|TLS/i', $r['err'])) { + $r['insecure'] = vv_auth_http_probe('https://' . $d . '/', false); + } + $out['live'][$d] = $r; + } + + $out['findings'] = vv_npm_why_findings($out); + return $out; +} + +// The deterministic half of the answer, written as sentences rather than codes. +// +// Separate from the gathering so it can be read, argued with and corrected on its own — and so the +// dialog has something certain to show whether or not there is a model on this node to interpret +// it. Most low figures on this installation have one of these causes, and none of them needs a +// language model to reach. +// +// Ordered most decisive first: the caller shows them in order and the first line is meant to be the +// answer. Each entry is ['level' => bad|warn|info, 'text' => …]. +function vv_npm_why_findings(array $w): array { + $f = []; + $up = $w['upstream'] ?? []; + $tcp = $up['tcp'] ?? []; + $cont = $up['container'] ?? null; + + // Whether the front door is failing *now*, decided once. Several readings below change meaning + // entirely on it — "the application answers directly" is a useful clue during an outage and a + // false alarm when the site is simply working, and an earlier draft said the second as if it + // were the first. + $frontBad = false; + foreach ($w['live'] ?? [] as $r) + if (($r['code'] ?? 0) === 0 || ($r['code'] ?? 0) >= 500) $frontBad = true; + + if (!($w['enabled'] ?? true)) + $f[] = ['level' => 'bad', 'text' => 'This host is disabled in NPM, so nothing is being served for it. The probe still counts it as unreachable.']; + + if ($cont && !($cont['running'] ?? true)) + $f[] = ['level' => 'bad', 'text' => 'Container ' . $cont['name'] . ' is ' . ($cont['status'] ?? 'not running') + . ' — nothing can answer on ' . ($up['host'] ?? '') . ':' . ($up['port'] ?? '') . '.']; + + // The direct check, stated as what it is. Whether it could be made at all is reported first, + // because a check that did not happen must never be read as a check that failed — see + // vv_auth_probe_target(). Most forward targets on this machine are docker names this process + // cannot resolve, and an earlier draft of this reported every one of them as a dead service. + $reached = ($up['probed'] ?? '') !== '' && (($tcp['ok'] ?? false) || !str_contains((string) ($tcp['err'] ?? ''), 'getaddrinfo')); + if (!$reached) { + $f[] = ['level' => 'info', 'text' => ($up['note'] ?: 'The forward target could not be resolved from here, so no direct check was made.') + . ' Nothing below is evidence that the service is down.']; + } elseif (!($tcp['ok'] ?? false)) { + $f[] = ['level' => 'bad', 'text' => 'Nothing is listening on ' . ($up['probed'] ?? '') . ':' . ($up['port'] ?? '') + . ' — ' . ($tcp['err'] ?? 'no reason given') + . ($cont && ($cont['running'] ?? false) + ? '. Container ' . $cont['name'] . ' is running, so the container is up and the application inside it is not serving that port.' + : '. The proxy has nothing to forward to.')]; + } else { + $uh = $up['http'] ?? null; + if ($uh && $uh['code'] >= 500) + $f[] = ['level' => 'bad', 'text' => 'The service is listening on ' . ($up['probed'] ?? '') . ':' . ($up['port'] ?? '') + . ' and answered HTTP ' . $uh['code'] . ' itself. The proxy is forwarding correctly — the fault is inside the application.']; + elseif ($uh && $uh['code'] === 0 && ($uh['err'] ?? '') !== '') + $f[] = ['level' => 'warn', 'text' => 'The port on ' . ($up['probed'] ?? '') . ' is open but nothing came back over it (' . $uh['err'] + . '). Something is holding the socket without serving — which is what the proxy sees as a timeout.']; + elseif ($uh && $frontBad) + $f[] = ['level' => 'info', 'text' => 'The service answers directly on ' . ($up['probed'] ?? '') . ':' . ($up['port'] ?? '') + . ' with HTTP ' . $uh['code'] . ', so whatever is failing sits between the proxy and it, not in the application.']; + } + + // The certificate, from both directions: what NPM says about its expiry, and what a live + // handshake actually did. Either can be the fault on its own — a cert with weeks left still + // fails if the chain it is serving is wrong. + $c = $w['cert'] ?? null; + if ($c && $c['days_left'] !== null) { + if ($c['days_left'] < 0) + $f[] = ['level' => 'bad', 'text' => 'The certificate expired ' . abs($c['days_left']) . ' days ago. Every HTTPS request to this host fails verification.']; + elseif ($c['days_left'] <= 14) + $f[] = ['level' => 'warn', 'text' => 'The certificate expires in ' . $c['days_left'] . ' days — check the Certs tab for whether renewal is running.']; + } + + foreach ($w['live'] ?? [] as $dom => $r) { + if (isset($r['insecure']) && $r['insecure']['code'] > 0 && $r['insecure']['code'] < 500) { + $f[] = ['level' => 'bad', 'text' => $dom . ' answers normally when certificate verification is turned off. ' + . 'The service is up and the certificate is what is failing: ' . $r['err']]; + } elseif ($r['code'] === 0 && ($r['err'] ?? '') !== '') { + $f[] = ['level' => 'bad', 'text' => $dom . ' did not answer just now — ' . $r['err']]; + } elseif ($r['code'] === 502 || $r['code'] === 504) { + // NPM's own verdict, and the one piece of evidence that is always authoritative: it is + // the component that actually has to reach the upstream. Paired with a direct probe + // that succeeded, it stops being "the app is down" and becomes a routing problem — + // the proxy and the application are on networks that cannot see each other. + $ok = ($up['http']['code'] ?? 0) > 0 && ($up['http']['code'] ?? 0) < 500; + $f[] = ['level' => 'bad', 'text' => $dom . ' answered HTTP ' . $r['code'] . ' through the proxy — NPM could not reach ' + . ($up['host'] ?? '') . ':' . ($up['port'] ?? '') . '.' + . ($ok ? ' It answers fine when asked directly, so the two are not on a network that can see each other,' + . ' or the forward host is written in a form NPM cannot resolve.' : '')]; + } elseif ($r['code'] >= 500) { + $f[] = ['level' => 'bad', 'text' => $dom . ' answered HTTP ' . $r['code'] . ' through the proxy.']; + } + } + + // Traffic, read against whether the host is guarded. An unguarded host that is nothing but 4xx + // is broken; a guarded one that is nothing but 4xx is usually Authelia doing its job to + // unauthenticated callers, and calling that a fault would send someone to fix what is working. + $t = $w['traffic'] ?? null; + if ($t && ($t['requests'] ?? 0) > 0) { + $req = (int) $t['requests']; + $s5 = (int) ($t['s5xx'] ?? 0); + $s4 = (int) ($t['s4xx'] ?? 0); + if ($s5 / $req >= 0.5) + $f[] = ['level' => 'bad', 'text' => round($s5 / $req * 100) . '% of all logged requests to this host are 5xx — this has been failing for real users, not just the probe.']; + if ($s4 / $req >= 0.9) + $f[] = $w['guarded'] + ? ['level' => 'info', 'text' => round($s4 / $req * 100) . '% of requests are 4xx, which on a host behind auth_request is usually Authelia refusing unauthenticated callers rather than a fault.'] + : ['level' => 'warn', 'text' => round($s4 / $req * 100) . '% of requests are 4xx and nothing is guarding this host, so callers are being refused by the application itself.']; + } + + // Shape of the loss. Same percentage, two entirely different problems, and this is the only + // reading that separates them. + foreach ($w['history'] ?? [] as $dom => $h) { + $ev = $h['events'] ?? []; + // Over what period, not just how many. Four state changes is flapping if they were this + // afternoon and completely unremarkable if they were spread across a month, and a count on + // its own cannot tell those apart — the store keeps the last twenty however old they are. + $span = count($ev) >= 2 ? ((int) ($ev[0]['ts'] ?? 0) - (int) ($ev[count($ev) - 1]['ts'] ?? 0)) : 0; + $spanTxt = $span >= 172800 ? 'over ' . round($span / 86400) . ' days' + : ($span >= 7200 ? 'over ' . round($span / 3600) . ' hours' : 'within the hour'); + if (count($ev) >= 4) + $f[] = ['level' => 'warn', 'text' => $dom . ' changed state ' . count($ev) . ' times ' . $spanTxt + . ' — this is flapping rather than one clean outage, so look for something restarting on a cycle.']; + elseif (count($h['bad_hours'] ?? []) === 1 && ($h['state'] ?? '') === 'up') + $f[] = ['level' => 'info', 'text' => $dom . ' lost samples in one hour only (' . $h['bad_hours'][0]['up'] . '/' . $h['bad_hours'][0]['total'] + . ' at ' . substr((string) $h['bad_hours'][0]['hour'], 8, 2) . ':00) and has been up since. A single event, already over.']; + } + + if (!$f) + $f[] = ['level' => 'info', 'text' => 'Nothing is failing right now — the service answers, the certificate verifies and the proxy is forwarding. Whatever cost this host its uptime is in the history below and has already ended.']; + + return $f; +} + // ── lldap ───────────────────────────────────────────────────────────────────── function vv_lldap_token(): string { @@ -569,9 +964,14 @@ function vv_lldap_remove_from_group(string $userId, int $groupId): array { // ── Authelia ────────────────────────────────────────────────────────────────── -function vv_authelia_read_rules(): array { +// $file overrides the configured path. There is more than one Authelia on this machine — the +// primary serves the .com names and Authelia-Secondary serves the .us ones — and conf names only +// the primary, so anything reasoning about a specific domain has to be able to read the instance +// that domain actually talks to. Defaults to the configured one, so every existing caller and the +// whole editing path are unchanged. +function vv_authelia_read_rules(?string $file = null): array { $conf = vv_auth_conf(); - $file = $conf['authelia_config']; + $file = $file ?: $conf['authelia_config']; if (!file_exists($file)) return ['ok' => false, 'error' => 'Config not found: ' . $file]; $content = file_get_contents($file); @@ -791,3 +1191,475 @@ function vv_authelia_yaml_scalar(string $val): string { return '"' . str_replace(['\\', '"'], ['\\\\', '\\"'], $val) . '"'; return $val; } + +// ── Access simulation ───────────────────────────────────────────────────────── +// +// Whether one person can open one URL is decided by three objects that no single card shows: the +// NPM host (does it hand the request to Authelia at all), the Authelia rule list (which rule wins, +// in file order), and the LDAP group membership (does the winning rule's subject include them). +// Any one of the three can be the reason a login is refused — or worse, not asked for — and the +// only way to find out has been to read three configs and reason about them by hand. +// +// This walks it the way Authelia does and reports every step, so the answer is checkable rather +// than asserted. It changes nothing: every function below reads. + +// Which Authelia a proxy host actually talks to, read out of its own nginx block rather than +// assumed from conf. +// +// This machine runs two — Authelia for the .com names and Authelia-Secondary for the .us ones — +// and HOST1_AUTHELIA_CONFIG names only the first. Evaluating a .us domain against the primary's +// rules would produce a confident, wrong answer for six live hostnames, so the instance is taken +// from the `set $upstream_authelia http://NAME:PORT` line that decides it in production. +function vv_authelia_instance_for(array $proxyHost): array { + $adv = (string) ($proxyHost['advanced_config'] ?? ''); + if (!str_contains($adv, 'auth_request')) + return ['guarded' => false, 'container' => '', 'config' => '', 'source' => 'none']; + + $container = ''; + if (preg_match('#set\s+\$upstream_authelia\s+https?://([A-Za-z0-9._-]+):(\d+)#', $adv, $m)) + $container = $m[1]; + + $conf = vv_auth_conf(); + // The configured instance is matched by name rather than assumed, so the tab's own editing + // target is identified as such and anything else is reported as the separate instance it is. + if ($container !== '' && strcasecmp($container, (string) ($conf['authelia_container'] ?? '')) === 0) + return ['guarded' => true, 'container' => $container, 'config' => $conf['authelia_config'], + 'source' => 'conf', 'is_configured' => true]; + + $path = $container !== '' ? vv_authelia_config_for_container($container) : ''; + return ['guarded' => true, 'container' => $container, 'config' => $path, + 'source' => $path !== '' ? 'docker' : 'unknown', 'is_configured' => false]; +} + +// A container's configuration.yml, found through its own /config bind mount. Nothing hardcodes a +// path: a second instance added later is picked up because it is mounted the same way, which is the +// same reason the rest of this plugin reads its host list from NPM rather than from conf. +function vv_authelia_config_for_container(string $name): string { + static $cache = []; + if (isset($cache[$name])) return $cache[$name]; + $cache[$name] = ''; + + require_once __DIR__ . '/docker.php'; + $all = vv_dk_inspect_all(); + foreach ($all as $cn => $c) { + if (strcasecmp($cn, $name) !== 0) continue; + foreach ($c['mounts'] ?? [] as $m) { + if (($m['dst'] ?? '') !== '/config') continue; + $p = rtrim((string) $m['src'], '/') . '/configuration.yml'; + if (is_file($p)) $cache[$name] = $p; + } + } + return $cache[$name]; +} + +// Does an Authelia domain pattern match this hostname? Authelia accepts an exact name and a single +// leading wildcard label; nothing else here uses regex domains, so nothing else is claimed. +function vv_authelia_domain_matches(string $pattern, string $domain): bool { + $pattern = strtolower(trim($pattern)); + $domain = strtolower(trim($domain)); + if ($pattern === '' || $domain === '') return false; + if ($pattern === $domain) return true; + if (str_starts_with($pattern, '*.')) { + $suffix = substr($pattern, 1); // ".example.com" + // One label only, matching Authelia: *.example.com covers a.example.com and not a.b.example.com. + return str_ends_with($domain, $suffix) + && !str_contains(substr($domain, 0, -strlen($suffix)), '.') + && substr($domain, 0, -strlen($suffix)) !== ''; + } + return false; +} + +// Whether a rule's subject admits this user. Authelia's shape is a list of subjects OR'd together, +// where an element that is itself a list is AND'd — so [[a,b],c] means "(a and b) or c". +// +// A rule with no subject at all applies to everyone, which is the case that silently shadows every +// specific rule below it. Returned as a reason string as well as a verdict, because "the rule was +// skipped" and "the rule matched and denied" look identical in a result and mean opposite things. +function vv_authelia_subject_matches($subject, string $uid, array $groups): array { + if ($subject === null || $subject === '' || $subject === []) + return ['match' => true, 'why' => 'no subject — applies to everyone']; + + $groupsLc = array_map('strtolower', $groups); + $one = function (string $s) use ($uid, $groupsLc): bool { + $s = trim($s); + if (str_starts_with($s, 'group:')) return in_array(strtolower(substr($s, 6)), $groupsLc, true); + if (str_starts_with($s, 'user:')) return strcasecmp(substr($s, 5), $uid) === 0; + // An unprefixed subject is a username in Authelia's schema. + return strcasecmp($s, $uid) === 0; + }; + + $alternatives = is_array($subject) ? $subject : [$subject]; + foreach ($alternatives as $alt) { + if (is_array($alt)) { + $all = true; + foreach ($alt as $part) if (!$one((string) $part)) { $all = false; break; } + if ($all) return ['match' => true, 'why' => 'matches all of ' . implode(' + ', array_map('strval', $alt))]; + } elseif ($one((string) $alt)) { + return ['match' => true, 'why' => 'matches ' . $alt]; + } + } + $flat = []; + foreach ($alternatives as $alt) $flat[] = is_array($alt) ? '(' . implode(' + ', array_map('strval', $alt)) . ')' : (string) $alt; + return ['match' => false, 'why' => 'not ' . implode(' or ', $flat)]; +} + +// Walk the rules in file order and stop at the first that matches on every axis, which is exactly +// what Authelia does. Every rule considered is reported with why it did or did not apply — the +// trace is the point, because "which rule won" is rarely the surprising part. "Which rule you +// thought would win and why it was skipped" is. +function vv_authelia_evaluate(array $rules, string $defaultPolicy, string $domain, string $path, + string $uid, array $groups): array { + $trace = []; + foreach ($rules as $i => $r) { + $doms = is_array($r['domain'] ?? '') ? $r['domain'] : [$r['domain'] ?? '']; + $domHit = false; + foreach ($doms as $d) if (vv_authelia_domain_matches((string) $d, $domain)) { $domHit = true; break; } + $label = trim(preg_replace('/^#+\s*/', '', implode(' ', $r['_label'] ?? []))) ?: ('rule ' . ($i + 1)); + + if (!$domHit) { $trace[] = ['n' => $i + 1, 'label' => $label, 'applied' => false, 'skip' => 'domain', 'why' => 'domain not listed']; continue; } + + // Resources is a path regex. A rule carrying one only applies to the paths it names, so a + // rule that looks like it covers a host may cover one directory of it. + $res = $r['resources'] ?? null; + if ($res !== null && $res !== '' && $res !== []) { + $list = is_array($res) ? $res : [$res]; + $hit = false; + foreach ($list as $rx) { + // Delimited and error-suppressed: this pattern comes from a hand-edited file, and a + // malformed one must report as "did not match" rather than raising a warning into + // the answer. + if (@preg_match('#' . str_replace('#', '\#', (string) $rx) . '#', $path)) { $hit = true; break; } + } + if (!$hit) { + $trace[] = ['n' => $i + 1, 'label' => $label, 'applied' => false, 'skip' => 'path', + 'why' => 'domain matches but the path ' . $path . ' is outside its resources pattern']; + continue; + } + } + + $sub = vv_authelia_subject_matches($r['subject'] ?? null, $uid, $groups); + if (!$sub['match']) { + // Recorded as a subject skip specifically. "No rule mentioned this host" and "a rule + // for this host stepped over this person" both end at the default policy and are + // completely different facts about the configuration. + $trace[] = ['n' => $i + 1, 'label' => $label, 'applied' => false, 'skip' => 'subject', + 'policy' => $r['policy'] ?? 'deny', + 'why' => 'domain matches but the user is ' . $sub['why']]; + continue; + } + + $trace[] = ['n' => $i + 1, 'label' => $label, 'applied' => true, + 'why' => 'domain matches and the user ' . $sub['why'], + 'policy' => $r['policy'] ?? 'deny']; + return ['policy' => $r['policy'] ?? 'deny', 'matched' => $i + 1, 'matched_label' => $label, 'trace' => $trace]; + } + return ['policy' => $defaultPolicy, 'matched' => null, 'matched_label' => '', 'trace' => $trace]; +} + +// The whole question, end to end: can this user open this URL, and what decided it. +function vv_auth_access_check(string $domain, string $uid, string $path = '/'): array { + $domain = strtolower(trim($domain)); + $path = $path === '' ? '/' : $path; + if ($domain === '') return ['ok' => false, 'error' => 'No domain given']; + + $p = vv_npm_list_proxies(); + if (!($p['ok'] ?? false)) return ['ok' => false, 'error' => $p['error'] ?? 'Could not read proxy hosts']; + + $host = null; + foreach ($p['proxies'] as $h) + foreach ($h['domain_names'] ?? [] as $d) + if (vv_authelia_domain_matches((string) $d, $domain) || strtolower((string) $d) === $domain) { $host = $h; break 2; } + + $out = ['ok' => true, 'domain' => $domain, 'path' => $path, 'uid' => $uid, 'findings' => []]; + + if (!$host) { + $out['findings'][] = ['level' => 'warn', 'text' => 'No NPM proxy host serves ' . $domain + . ', so nothing reaches Authelia for it and no rule about it has any effect.']; + $out['served'] = false; + return $out; + } + $out['served'] = true; + $out['enabled'] = ($host['enabled'] ?? true) ? true : false; + $out['forward'] = ($host['forward_scheme'] ?? 'http') . '://' . ($host['forward_host'] ?? '') . ':' . ($host['forward_port'] ?? ''); + + // The user's groups, which is the half of the answer that lives in a different system entirely. + // Taken from the user record rather than by walking every group, because that record carries + // both facts this needs — whether the person exists and what they belong to. Asking the group + // list instead would make "in no groups" and "no such person" the same empty result. + $groups = []; + $known = false; + if ($uid !== '') { + $ul = vv_lldap_list_users(); + foreach (($ul['ok'] ?? false) ? ($ul['users'] ?? []) : [] as $u) { + if (strcasecmp((string) ($u['id'] ?? ''), $uid) !== 0) continue; + $known = true; + $groups = array_values(array_filter(array_column($u['groups'] ?? [], 'displayName'))); + break; + } + } + $out['groups'] = $groups; + $out['user_known'] = $known; + + $inst = vv_authelia_instance_for($host); + $out['authelia'] = $inst; + + if (!$inst['guarded']) { + $out['policy'] = 'bypass'; + $out['findings'][] = ['level' => 'warn', 'text' => 'This host has no auth_request block, so the request never reaches Authelia. ' + . 'Anyone who can resolve ' . $domain . ' gets through to the application, whatever the rules say.']; + return $out; + } + if ($inst['config'] === '' || !is_file($inst['config'])) { + $out['findings'][] = ['level' => 'warn', 'text' => 'This host sends its authentication to ' . ($inst['container'] ?: 'an unnamed instance') + . ', whose configuration could not be located, so the decision below cannot be worked out.']; + return $out; + } + + $r = vv_authelia_read_rules($inst['config']); + if (!($r['ok'] ?? false)) { $out['findings'][] = ['level' => 'warn', 'text' => $r['error'] ?? 'Rules unreadable']; return $out; } + + $ev = vv_authelia_evaluate($r['rules'] ?? [], $r['default_policy'] ?? 'deny', $domain, $path, $uid, $groups); + $out['policy'] = $ev['policy']; + $out['matched'] = $ev['matched']; + $out['matched_label'] = $ev['matched_label']; + $out['trace'] = $ev['trace']; + $out['default_policy'] = $r['default_policy'] ?? 'deny'; + $out['findings'] = array_merge($out['findings'], vv_auth_access_findings($out, $inst)); + return $out; +} + +function vv_auth_access_findings(array $o, array $inst): array { + $f = []; + + // The one that cannot be seen from any single page. A rule list that is not the one being + // edited on this tab is a rule list nobody is maintaining on purpose. + if (!($inst['is_configured'] ?? false)) + $f[] = ['level' => 'info', 'text' => 'Decided by ' . ($inst['container'] ?: 'a second instance') + . ', which is not the Authelia this tab edits. Its rules are in ' . ($inst['config'] ?: 'a config that was not found') + . ' and nothing on this page changes them.']; + + if (!($o['enabled'] ?? true)) + $f[] = ['level' => 'warn', 'text' => 'The proxy host is disabled in NPM, so nothing is served here at all right now.']; + + if ($o['uid'] !== '' && !($o['user_known'] ?? false)) + $f[] = ['level' => 'warn', 'text' => 'No user with the id "' . $o['uid'] . '" exists in the directory, so this is the answer for a name that cannot log in.']; + elseif ($o['uid'] !== '' && !$o['groups']) + $f[] = ['level' => 'info', 'text' => $o['uid'] . ' is in no groups, so every rule with a group subject skips them.']; + + $policy = $o['policy'] ?? ''; + if ($o['matched'] === null) { + // Two different facts end at the same default policy, and conflating them was the first + // version of this: a host no rule mentions, and a host whose rule stepped over this + // particular person. The second is the more pointed one — the rule exists, it was written + // for this host, and the default let them past it anyway. + $stepped = []; + foreach ($o['trace'] ?? [] as $t) if (($t['skip'] ?? '') === 'subject') $stepped[] = $t; + + if ($stepped && $policy === 'bypass') { + $t = $stepped[0]; + $f[] = ['level' => 'bad', 'text' => 'Rule ' . $t['n'] . ' (' . $t['label'] . ') covers ' . $o['domain'] + . ' but does not apply to this user — ' . $t['why'] . '. No later rule matches either, so the default policy takes over, ' + . 'and the default here is bypass. The rule written to protect this host lets everyone it does not name straight through.']; + // Stated because the answer is different for a caller who is not logged in at all, and + // an operator reading "bypass" would otherwise reasonably conclude the host is open to + // the internet. Authelia treats an anonymous request against a rule carrying a subject + // as a potential match and sends them to the portal first; this simulation answers for + // someone who has already authenticated as this user. + $f[] = ['level' => 'info', 'text' => 'This is the answer for a caller already logged in as ' . ($o['uid'] ?: 'someone') + . '. Authelia handles an anonymous caller differently — a rule carrying a subject makes it send them to the login portal first — ' + . 'so this is an authenticated user reaching something not meant for them, not an open door to the internet.']; + } elseif ($policy === 'bypass') { + $f[] = ['level' => 'bad', 'text' => 'No rule mentions ' . $o['domain'] . ' at all, so it falls to the default policy, which is bypass — ' + . 'the request goes to Authelia and Authelia waves it through. This host is behind an auth_request block that never refuses anyone.']; + } else { + $f[] = ['level' => 'info', 'text' => 'No rule matches ' . $o['domain'] . ', so the default policy of ' . $policy . ' applies.']; + } + } else { + $f[] = ['level' => $policy === 'deny' ? 'warn' : 'info', + 'text' => 'Rule ' . $o['matched'] . ' (' . $o['matched_label'] . ') is the first one that applies, and its policy is ' . $policy . '.']; + } + + if ($policy === 'bypass' || $policy === '') + $f[] = ['level' => 'info', 'text' => ($o['uid'] !== '' ? $o['uid'] : 'Anyone') . ' reaches ' . $o['domain'] + . ' without being asked to authenticate. Whether that is right depends on whether the application behind it has its own login.']; + elseif ($policy === 'deny') + $f[] = ['level' => 'warn', 'text' => ($o['uid'] !== '' ? $o['uid'] : 'This caller') . ' is refused before reaching the application.']; + else + $f[] = ['level' => 'info', 'text' => ($o['uid'] !== '' ? $o['uid'] : 'A caller') . ' is asked to log in (' . $policy . ') and then reaches the application.']; + + return $f; +} + +// ── Certificate renewal triage ──────────────────────────────────────────────── +// +// Why renewals failed, from certbot's own logs. cert_history.sh counts failures by noticing an +// expiry in the past; this reads the reason. See Tools/cert_triage.php for the full note. + +// The categories renewal failures actually fall into here, in the order a reader should meet them: +// causes before consequences. Each pattern is anchored on the string certbot itself emits, so a +// category matching is evidence rather than inference. +// +// 'root' marks a cause worth acting on directly. Rate limiting is deliberately not one — it is +// what happens after something else has been failing, and treating it as the problem sends people +// to wait out a timer instead of fixing the DNS record that burned it. +const VV_CERT_TRIAGE_PATTERNS = [ + ['id' => 'no-dns-record', 'root' => true, + 'rx' => '/DNS problem: NXDOMAIN looking up [A-Z]+ for ([A-Za-z0-9._-]+)/', + 'what' => 'the hostname has no DNS record at all'], + ['id' => 'no-a-record', 'root' => true, + 'rx' => '/no valid A records found for ([A-Za-z0-9._-]+)/', + 'what' => 'the hostname resolves but has no address record Let\'s Encrypt can reach'], + ['id' => 'challenge-unreachable', 'root' => true, + 'rx' => '/Timeout during connect[^\n]*|Fetching http:\/\/([A-Za-z0-9._-]+)\/\.well-known[^\n]*Timeout/', + 'what' => 'the HTTP-01 challenge could not be fetched — port 80 is not reaching this proxy'], + ['id' => 'caa-forbids', 'root' => true, + 'rx' => '/CAA record for ([A-Za-z0-9._-]+) prevents issuance/', + 'what' => 'a CAA record on the domain forbids Let\'s Encrypt from issuing'], + ['id' => 'revoke-expired', 'root' => false, + 'rx' => '/Unable to revoke :: Certificate is expired/', + 'what' => 'a revoke was attempted on a certificate that had already expired'], + // Last, and marked as a consequence. 2079 of these on this installation, every one of them + // downstream of the three hostnames above. + ['id' => 'rate-limited', 'root' => false, + 'rx' => '/urn:ietf:params:acme:error:rateLimited/', + 'what' => 'Let\'s Encrypt refused the request because too many were made too recently'], +]; + +// The rotated certbot logs, newest first, ordered by their rotation suffix. +// +// Never by mtime. Every one of these files carries the same mtime on this machine — they live in +// the Critical-Data share and are written as a set by the sync, so the filesystem says all +// thousand were modified in the same minute. Sorting by mtime picks an arbitrary sample from +// anywhere in the history and presents it as the current state. +function vv_cert_log_files(int $limit): array { + $dir = trim((string) (vv_conf_vars()['CERT_TRIAGE_LOG_DIR'] ?? '')); + if ($dir === '') { + require_once __DIR__ . '/docker.php'; + foreach (vv_dk_inspect_all() as $name => $c) { + if (stripos($name, 'nginx') === false && stripos($name, 'npm') === false) continue; + foreach ($c['mounts'] ?? [] as $m) { + if (($m['dst'] ?? '') !== '/config' && ($m['dst'] ?? '') !== '/data') continue; + $p = rtrim((string) $m['src'], '/') . '/log'; + if (is_dir($p)) { $dir = $p; break 2; } + } + } + } + if ($dir === '' || !is_dir($dir)) return ['dir' => $dir, 'files' => []]; + + $found = glob($dir . '/letsencrypt.log*') ?: []; + $rank = []; + foreach ($found as $f) { + // letsencrypt.log is the live one and sorts ahead of every numbered rotation. + $n = preg_match('/\.log\.(\d+)$/', $f, $m) ? (int) $m[1] : -1; + $rank[$f] = $n; + } + asort($rank); + return ['dir' => $dir, 'files' => array_slice(array_keys($rank), 0, max(1, $limit))]; +} + +// The tail of a file, without reading the whole thing. These run to a megabyte each and the +// interesting part of a certbot run is always at the end. +function vv_cert_log_tail(string $path, int $maxBytes): string { + $size = @filesize($path); + if ($size === false) return ''; + $fh = @fopen($path, 'rb'); + if (!$fh) return ''; + if ($size > $maxBytes) @fseek($fh, $size - $maxBytes); + $data = (string) @stream_get_contents($fh); + fclose($fh); + return $data; +} + +function vv_cert_triage(int $filesOverride = 0): array { + $v = vv_conf_vars(); + $lim = $filesOverride > 0 ? $filesOverride : max(1, (int) ($v['CERT_TRIAGE_FILES'] ?? 40)); + $max = max(4096, (int) ($v['CERT_TRIAGE_MAX_BYTES'] ?? 262144)); + + $found = vv_cert_log_files($lim); + if (!$found['files']) + return ['ok' => false, 'error' => 'No certbot logs found' + . ($found['dir'] !== '' ? ' in ' . $found['dir'] : ' — set CERT_TRIAGE_LOG_DIR')]; + + // Counted per run, not per line. One log file is one certbot invocation, and a single failed + // run writes its reason several times over — in the ACME response, in the traceback, and again + // in certbot's own ERROR summary. Counting lines therefore reports one failure as three and + // makes the categories incomparable with each other, because the noisier reasons repeat more. + // "12 of 40 runs failed for no DNS record" is a number that means something. + $counts = $doms = []; + $runs = $failedRuns = $unclassified = 0; + + foreach ($found['files'] as $f) { + $text = vv_cert_log_tail($f, $max); + if ($text === '') continue; + $runs++; + + $hitAny = false; + foreach (VV_CERT_TRIAGE_PATTERNS as $p) { + if (!preg_match_all($p['rx'], $text, $m, PREG_SET_ORDER)) continue; + $hitAny = true; + $counts[$p['id']] = ($counts[$p['id']] ?? 0) + 1; + foreach ($m as $hit) + // Only some patterns capture a hostname; the others are about the run, not a name. + if (isset($hit[1]) && $hit[1] !== '') $doms[$p['id']][strtolower($hit[1])] = true; + } + if ($hitAny) $failedRuns++; + + // A run that errored and matched nothing known. Reported rather than dropped: an error + // certbot starts emitting after this was written has to show up as something, and a total + // that quietly shrinks is how a new failure mode stays invisible. + if (!$hitAny && preg_match('/:ERROR:certbot/', $text)) { $unclassified++; $failedRuns++; } + } + + $cats = []; + foreach (VV_CERT_TRIAGE_PATTERNS as $p) { + if (empty($counts[$p['id']])) continue; + $cats[] = ['id' => $p['id'], 'root' => $p['root'], 'what' => $p['what'], + 'count' => $counts[$p['id']], 'domains' => array_keys($doms[$p['id']] ?? [])]; + } + + return ['ok' => true, 'dir' => $found['dir'], 'files_read' => $runs, + 'total' => $failedRuns, 'unclassified' => $unclassified, + 'categories' => $cats, 'reading' => vv_cert_triage_reading($cats)]; +} + +// The causal reading, which is the part a category count cannot give. Written as sentences because +// the relationship between these categories is the whole finding: one of them is nearly always +// downstream of another, and a list sorted by count puts the consequence at the top. +function vv_cert_triage_reading(array $cats): array { + if (!$cats) return []; + $by = []; + foreach ($cats as $c) $by[$c['id']] = $c; + + $roots = array_values(array_filter($cats, fn($c) => $c['root'])); + $out = []; + + if ($roots) { + $names = []; + foreach ($roots as $r) foreach ($r['domains'] as $d) $names[$d] = true; + $n = count($roots); + $out[] = 'Root cause: ' . $n . ' kind' . ($n === 1 ? '' : 's') . ' of failure that ' + . ($n === 1 ? 'is' : 'are') . ' nobody else\'s consequence' + . ($names ? ', affecting ' . implode(', ', array_slice(array_keys($names), 0, 8)) + . (count($names) > 8 ? ' and ' . (count($names) - 8) . ' more' : '') : '') . '.'; + } + + if (isset($by['rate-limited'])) { + $out[] = $roots + ? 'The ' . $by['rate-limited']['count'] . ' rate-limit refusals are downstream of that: ' + . 'certbot retried the failing names often enough to exhaust the allowance, which then ' + . 'fails renewals for domains that have nothing wrong with them. Fixing the names above ' + . 'is what clears it — waiting out the limit only restarts the cycle.' + : 'Rate limiting is the only category present, with no failing name behind it. That points ' + . 'at renewal being attempted far too often rather than at any one domain.'; + } + + if (isset($by['revoke-expired'])) + $out[] = 'The revoke failures are harmless in themselves — a certificate that already expired ' + . 'cannot be revoked, and does not need to be. They indicate cleanup running against ' + . 'certificates that were already dead.'; + + if (!$roots && !isset($by['rate-limited'])) + $out[] = 'Nothing here is a standing cause — these are individual failures rather than a pattern.'; + + return $out; +} diff --git a/Plugin/unraid/pages/auth.php b/Plugin/unraid/pages/auth.php index 652088a..263e253 100644 --- a/Plugin/unraid/pages/auth.php +++ b/Plugin/unraid/pages/auth.php @@ -238,8 +238,10 @@ require_once dirname(__DIR__) . '/include/ai_chat.php'; /* ── Misc ────────────────────────────────────────────────────────────────── */ .vv-au-empty { padding:24px;text-align:center;font-size:11px;color:#333; } -.vv-au-domain { font-size:12px;color:#bbb;font-weight:bold; } -.vv-au-fwd { font-size:10px;color:#444;font-family:monospace;margin-top:1px; } +/* The host column is the flexible one in the proxy table, so a long list of domains has to wrap + inside it rather than push the figure columns out of line. */ +.vv-au-domain { font-size:12px;color:#bbb;font-weight:bold;overflow-wrap:anywhere; } +.vv-au-fwd { font-size:10px;color:#444;font-family:monospace;margin-top:1px;overflow-wrap:anywhere; } /* ── Proxy row marks and traffic ─────────────────────────────────────────── */ /* Small marks rather than four more columns. Each one is a fact about the host that had no @@ -253,7 +255,7 @@ require_once dirname(__DIR__) . '/include/ai_chat.php'; .vv-au-m.auth { background:#1a0d2a;border-color:#2a1a4a;color:#9c6ff7;font-weight:600; } /* Requests and errors are their own columns now. Right-aligned so the magnitudes line up down the page — the whole point of the errors column is spotting the one host that is all failures. */ -.vv-au-num { text-align:right;white-space:nowrap;width:1%; } +.vv-au-num { text-align:right;white-space:nowrap; } .vv-au-stat-n { font-size:13px;font-weight:700;color:#bbb;line-height:1.15; } .vv-au-stat-n.dim { color:#2e2e2e;font-weight:normal;font-size:12px; } .vv-au-stat-n.ok { color:#2d4a2d; } @@ -270,6 +272,79 @@ require_once dirname(__DIR__) . '/include/ai_chat.php'; the two red ones do not. */ .vv-au-stat-n.ok2 { color:#4a7a4a; } +/* Fixed columns for the proxy table. With auto layout each figure column sized itself to its own + widest cell, so the three of them were three different widths and every column shifted whenever + the numbers changed. Declared once here and the host column takes whatever is left. */ +.vv-au-tbl.fixed { table-layout:fixed;min-width:960px; } +.vv-au-tblwrap { overflow-x:auto; } +.vv-au-c-flags { width:154px; } +/* 60 spark bars at 2px plus 1px gaps is 179px, and the cell has 10px padding either side. Every + figure column gets that same width whether or not it draws bars. */ +.vv-au-c-stat { width:200px; } +.vv-au-c-tog { width:56px; } +.vv-au-c-act { width:64px; } + +/* ── Why ─────────────────────────────────────────────────────────────────── */ +/* On the figure's own line, not under the strip: the strip is 179px of fixed-width bars and is what + sets this column's width, so anything below it would widen every row for the four that need it. */ +.vv-au-upl { display:flex;align-items:center;justify-content:flex-end;gap:6px; } +.vv-au-why { font-size:9px;padding:1px 5px;border-radius:2px;background:#1f1200; + border:1px solid #3a2000;color:#ff9800;cursor:pointer;line-height:1.4; } +.vv-au-why:hover{ background:#2a1900;color:#ffb74d; } +.vv-au-why:disabled { opacity:.4;cursor:default; } +.vv-au-why-wait { font-size:11px;color:#666;padding:14px 2px;line-height:1.5; } +/* The findings are the answer, so they get the top of the dialog and enough weight to be read as + sentences rather than as another row of status. */ +.vv-au-find { font-size:11px;line-height:1.5;padding:8px 10px;border-radius:3px;margin-bottom:6px; + border-left:2px solid #333;background:#141414;color:#888; } +.vv-au-find.bad { border-left-color:#ef5350;color:#d88; } +.vv-au-find.warn{ border-left-color:#ff9800;color:#b98a4a; } +.vv-au-find.info{ border-left-color:#2a4a6a;color:#777; } +.vv-au-why-grid { display:grid;grid-template-columns:auto 1fr;gap:3px 12px;margin:12px 0 4px;font-size:11px; } +.vv-au-why-k { color:#444;white-space:nowrap; } +.vv-au-why-v { color:#999;overflow-wrap:anywhere; } +.vv-au-why-sec { font-size:10px;color:#444;text-transform:uppercase;letter-spacing:.06em; + margin:14px 0 6px;border-top:1px solid #222;padding-top:10px; } +.vv-au-why-h { margin-bottom:10px; } +.vv-au-why-hd { font-size:11px;color:#999;display:flex;justify-content:space-between;gap:10px; } +.vv-au-why-bad { font-size:10px;color:#8a6a2a;margin:2px 0 3px; } +.vv-au-why-ev { font-size:10px;color:#666;margin-top:2px; } +.vv-au-why-note { font-size:10px;color:#4a4a4a;margin-top:2px;line-height:1.45; } + +/* ── Access simulator ────────────────────────────────────────────────────── */ +.vv-au-sim { padding:12px 14px;margin-bottom:10px; } +.vv-au-sim-row { display:flex;align-items:center;gap:8px;flex-wrap:wrap; } +.vv-au-sim-row .vv-au-label { margin-bottom:0; } +.vv-au-sim-row .vv-au-select { width:auto;min-width:150px;flex:0 1 auto; } +.vv-au-sim-path { width:110px;flex:0 0 auto; } +/* One word, because that is the question. Sized to be readable from the second screen. */ +.vv-au-sim-verdict { font-size:20px;font-weight:700;letter-spacing:.02em;margin:14px 0 10px; } +.vv-au-sim-verdict.ok { color:#4a7a4a; } +.vv-au-sim-verdict.warn { color:#ff9800; } +.vv-au-sim-verdict.bad { color:#ef5350; } +.vv-au-sim-t { display:flex;gap:8px;font-size:10px;color:#555;padding:3px 0;align-items:baseline; } +.vv-au-sim-t.hit{ color:#999; } +.vv-au-sim-n { color:#333;min-width:14px; } +.vv-au-sim-t.hit .vv-au-sim-n { color:#4a7a4a; } +.vv-au-sim-l { min-width:130px;color:#666; } +.vv-au-sim-t.hit .vv-au-sim-l { color:#bbb;font-weight:600; } +.vv-au-sim-w { color:#444;overflow-wrap:anywhere; } +.vv-au-sim-t.hit .vv-au-sim-w { color:#777; } + +/* ── Renewal triage ──────────────────────────────────────────────────────── */ +/* Root causes carry the marker; consequences are dimmed. Sorting these by count would put rate + limiting first every time, and rate limiting is the one thing here never worth fixing directly. */ +.vv-au-tri-row { display:flex;gap:10px;padding:6px 0;border-bottom:1px solid #1a1a1a;align-items:baseline; } +.vv-au-tri-row:last-of-type { border-bottom:none; } +.vv-au-tri-row.root .vv-au-tri-n { color:#ff9800; } +.vv-au-tri-n { font-size:15px;font-weight:700;color:#555;min-width:34px;text-align:right; } +.vv-au-tri-w { font-size:11px;color:#888;line-height:1.5; } +.vv-au-tri-d { font-size:10px;color:#4a4a4a;font-family:monospace;margin-top:2px;overflow-wrap:anywhere; } +.vv-au-ok { color:#4a7a4a; } +.vv-au-bad { color:#ef5350; } +.vv-au-warn { color:#ff9800; } +.vv-au-dim { color:#444; } + /* One bar per probe, last hour. The shape is the part a percentage throws away: one long outage and sixty scattered blips are the same 50% and completely different problems. */ .vv-au-spark { display:flex;gap:1px;justify-content:flex-end;align-items:flex-end;height:11px;margin-top:3px; } @@ -361,12 +436,18 @@ $tabs = ['proxies' => 'Proxies', 'users' => 'Users & Groups',
Loading…
- + + + + +
HostFlagsUptimeRequestsErrorsOn
+
@@ -418,6 +499,22 @@ $tabs = ['proxies' => 'Proxies', 'users' => 'Users & Groups', + +
+
+ Can + + open + + + +
+
+
@@ -437,8 +534,13 @@ $tabs = ['proxies' => 'Proxies', 'users' => 'Users & Groups',
" id="vv-au-panel-certs">
+ +
+
Loading…
@@ -536,8 +638,11 @@ function _esc(s) { return String(s||'').replace(/&/g,'&').replace(//g,'>').replace(/"/g,'"'); } -function _get(action, cb) { - fetch(API + '?action=' + action) +// params is optional and encoded rather than concatenated — the only caller passing one passes an +// id, but a query string built by hand is how a value with an & in it silently becomes two. +function _get(action, cb, params) { + const qs = params ? '&' + new URLSearchParams(params).toString() : ''; + fetch(API + '?action=' + action + qs) .then(r => r.json()).then(cb) .catch(e => cb({ ok: false, error: String(e) })); } @@ -639,7 +744,7 @@ function _loadTab(tab) { // ── Proxies ─────────────────────────────────────────────────────────────────── function _loadProxies() { const loading = document.getElementById('vv-au-proxy-loading'); - const tbl = document.getElementById('vv-au-proxy-tbl'); + const tbl = document.getElementById('vv-au-proxy-wrap'); const empty = document.getElementById('vv-au-proxy-empty'); loading.style.display = 'block'; tbl.style.display = 'none'; @@ -651,7 +756,7 @@ function _loadProxies() { if (!certsLoaded || !proxiesLoaded) return; loading.style.display = 'none'; if (!_proxies.length) { empty.style.display = 'block'; return; } - tbl.style.display = 'table'; + tbl.style.display = 'block'; _renderProxies(); } @@ -765,10 +870,189 @@ function _upCell(p) { ? 'DOWN — ' + (worst.last_detail || 'no response') : 'up' + (worst.last_ms ? ' · ' + worst.last_ms + 'ms' : ''); - return `
${pct === null ? '—' : (pct >= 99.95 ? '100' : pct.toFixed(pct >= 99 ? 2 : 1))}% 24h
+ // Offered only where there is something to explain. A why? on all thirty-five rows is a button + // nobody reads; on the four that are not fine it is the next thing you were going to do anyway. + // + // 96 rather than 100 because a probe every minute makes 99.9% an ordinary week — one missed + // sample in a day is 99.93 and means nothing. Below 96 is roughly an hour lost in a day, which is + // always something. Anything currently down qualifies whatever its percentage says. + const needWhy = down || (pct !== null && pct < 96); + const why = needWhy + ? `` + : ''; + + return `
${why}
${pct === null ? '—' : (pct >= 99.95 ? '100' : pct.toFixed(pct >= 99 ? 2 : 1))}% 24h
${bars}
`; } +// ── Why ─────────────────────────────────────────────────────────────────────── +// Held so the assistant hand-off has something to send without asking the server twice. Cleared +// with the dialog, because a brief describing a probe from ten minutes ago is worse than no brief. +let _why = null; + +function _whyModal(id, btn) { + const p = _proxies.find(x => x.id === id); + const name = (p?.domain_names || []).join(', ') || ('host ' + id); + _why = null; + // Disabled for the duration. The check takes seconds and opens sockets, and a second press while + // the first is in flight probes the host twice to answer the same question. + if (btn) btn.disabled = true; + + // Named while it works, because it genuinely takes a few seconds and a silent dialog reads as a + // hang. What it is about to do is listed rather than hidden behind a spinner — this opens + // connections, and an operator watching a limping host should know the page is about to touch it. + _modal(`

Why — ${_esc(name)}

+
Looking… connecting to the service behind this host, asking the + domain itself, and reading the probe history.
`, true); + + _get('npm_why', r => { + if (btn) btn.disabled = false; + if (!r || !r.ok) { + _modal(`

Why — ${_esc(name)}

+
${_esc(r?.error || 'The check failed')}
+
`, true); + document.getElementById('wm-close').onclick = _closeModal; + return; + } + _why = r; + _modal(_whyHtml(r, name), true); + document.getElementById('wm-close').onclick = _closeModal; + const ask = document.getElementById('wm-ask'); + if (ask) ask.onclick = () => _whyAsk(name); + }, { id }); +} + +// Hand the measurements to the assistant. The dialog closes first: the chat card is at the foot of +// the page and an answer streaming in behind an overlay is an answer nobody sees. +function _whyAsk(name) { + const chat = window.__vvAiChat && window.__vvAiChat['vv-au-ai']; + if (!chat || !_why) return; + if (chat.busy()) { vvAlert('The assistant is still answering the previous question.'); return; } + const brief = _whyBrief(_why, name); + _closeModal(); + // retarget, so the thread is about this host and not appended to whatever was asked before it. + // Same reasoning as the Watchdog tab: a troubleshooting thread about one host must not bleed into + // a question about another. + chat.retarget('troubleshoot', name, 'now looking at ' + name); + const input = document.getElementById('vv-au-ai-input'); + if (input) input.value = brief; + chat.send(); + const card = document.getElementById('vv-au-ai-card'); + if (card) card.scrollIntoView({ behavior: 'smooth', block: 'nearest' }); +} + +function _whyHtml(r, name) { + const lvl = { bad: 'bad', warn: 'warn', info: 'info' }; + + // The findings first and everything else under them. The evidence is what makes the answer + // checkable, but it is not the answer, and a dialog that opens on a table of status codes makes + // the reader do the work the check just did. + const findings = (r.findings || []).map(f => + `
${_esc(f.text)}
`).join(''); + + const up = r.upstream || {}; + const tcp = up.tcp || {}; + const rows = []; + + rows.push(['Forwards to', _esc(r.forward) + + (up.container ? ` ${_esc(up.container.name)} · ${_esc(up.container.status)}` : '')]); + // The address actually knocked on, not the one written in NPM. They differ whenever the forward + // host is a docker name, and a result attributed to the wrong address is worse than no result. + rows.push([tcp.ok ? 'Port open' : 'Port', (tcp.ok + ? `${_esc(up.probed || '')}:${up.port} ${tcp.ms}ms` + : `${_esc(up.probed || '')}:${up.port} — ${_esc(tcp.err || 'no answer')}`) + + (up.note ? `
${_esc(up.note)}
` : '')]); + if (up.http) + rows.push(['Service answers', up.http.code + ? `HTTP ${up.http.code} ${up.http.ms}ms` + : `${_esc(up.http.err || 'no response')}`]); + + // Both halves of the certificate: what NPM records and what the handshake did. They disagree more + // often than they should — a cert with a month left still fails if the chain being served is wrong. + if (r.cert) { + const d = r.cert.days_left; + rows.push(['Certificate', `${_esc(r.cert.name)} ` + + (d === null ? 'no expiry recorded' : (d < 0 ? `expired ${Math.abs(d)}d ago` : `${d}d left`)) + '']); + } + if (r.guarded) rows.push(['Protection', 'behind Authelia (auth_request)']); + if (!r.enabled) rows.push(['State', 'disabled in NPM']); + + for (const [d, l] of Object.entries(r.live || {})) { + const okNow = l.code > 0 && l.code < 500; + rows.push([_esc(d), (okNow ? `HTTP ${l.code} ${l.ms}ms` + : `${l.code ? 'HTTP ' + l.code : _esc(l.err || 'no answer')}`) + + (l.insecure ? ` — answers ${l.insecure.code} without certificate verification` : '')]); + } + + const t = r.traffic; + if (t) rows.push(['Logged traffic', `${_fmtNum(t.requests || 0)} requests · ${_fmtNum(t.s4xx || 0)} 4xx · ${_fmtNum(t.s5xx || 0)} 5xx` + + (t.last_seen ? ` · last ${_ago(t.last_seen)} ago` : ' · never hit')]); + + const grid = rows.map(([k, v]) => + `
${k}
${v}
`).join(''); + + // The recorded history, per domain. Kept last and kept short: it is the part that says whether + // this is happening now or already over, which only matters once you know what "this" is. + const hist = Object.entries(r.history || {}).map(([d, h]) => { + const ev = (h.events || []).map(e => + `
${_esc(String(e.to).toUpperCase())} + ${_ago(e.ts)} ago ${_esc(e.detail || '')}
`).join(''); + const bad = (h.bad_hours || []).map(b => + `${String(b.hour).slice(8, 10)}:00 ${b.up}/${b.total}`).join(' · '); + return `
+
${_esc(d)} + ${h.h1 === null ? '—' : h.h1 + '% 1h'} · ${h.h24 === null ? '—' : h.h24 + '% 24h'} · ${h.d30 === null ? '—' : h.d30 + '% 30d'}
+ ${bad ? `
Hours that lost samples: ${bad}
` : ''} + ${ev || '
no state changes recorded
'} +
`; + }).join(''); + + // Offered only where there is a model to ask. The findings above are the page's own reading and + // stand without it — the assistant is for the step after, which is what to do about it. + const ask = document.getElementById('vv-au-ai-chat') + ? '' : ''; + + return `

Why — ${_esc(name)}

+ ${findings} +
${grid}
+ ${hist ? `
Recorded history
${hist}` : ''} +
${ask}
`; +} + +// The brief the assistant is given. Deliberately the same facts the dialog just showed, written out +// rather than summarised: a model asked "why is this host down" with no evidence answers from the +// general shape of the question, and the whole point of the check above is that it went and looked. +function _whyBrief(r, name) { + const L = []; + L.push(`Proxy host ${name} on this machine is not at full uptime. Here is what the Auth tab just measured.`); + L.push(`Forwards to ${r.forward}${r.guarded ? ', behind Authelia via auth_request' : ''}${r.enabled ? '' : '. The host is DISABLED in NPM'}.`); + + const up = r.upstream || {}, tcp = up.tcp || {}; + if (up.note) L.push(`Note on the check itself: ${up.note}`); + L.push(`TCP to ${up.probed}:${up.port} — ${tcp.ok ? 'open in ' + tcp.ms + 'ms' : 'failed: ' + (tcp.err || 'no reason')}.`); + if (up.container) L.push(`That target is container ${up.container.name}, currently ${up.container.status}.`); + if (up.http) L.push(`Asked the service directly: ${up.http.code ? 'HTTP ' + up.http.code : 'no response (' + (up.http.err || '') + ')'}.`); + if (r.cert) L.push(`Certificate ${r.cert.name}: ${r.cert.days_left === null ? 'no expiry recorded' : r.cert.days_left + ' days left'}.`); + + for (const [d, l] of Object.entries(r.live || {})) + L.push(`Through the proxy, https://${d}/ — ${l.code ? 'HTTP ' + l.code + ' in ' + l.ms + 'ms' : 'no answer: ' + (l.err || '')}` + + (l.insecure ? `; with certificate verification off it answers ${l.insecure.code}` : '') + '.'); + + const t = r.traffic; + if (t) L.push(`Access log totals: ${t.requests || 0} requests, ${t.s4xx || 0} 4xx, ${t.s5xx || 0} 5xx.`); + + for (const [d, h] of Object.entries(r.history || {})) { + L.push(`${d}: ${h.h1 === null ? '?' : h.h1}% in the last hour, ${h.h24 === null ? '?' : h.h24}% over 24h, ${h.d30 === null ? '?' : h.d30}% over 30 days; currently ${h.state || 'unknown'}, last probe said "${h.last_detail || '?'}".`); + for (const e of (h.events || []).slice(0, 5)) + L.push(` went ${e.to} ${_ago(e.ts)} ago — ${e.detail || ''}`); + } + + L.push(''); + L.push('What is the most likely cause, and what should I check or change first? ' + + 'If the evidence above already settles it, say so rather than listing possibilities.'); + return L.join('\n'); +} + function _certName(id) { const c = _certs.find(x => String(x.id) === String(id)); return c ? (c.nice_name || (c.domain_names || []).join(', ')) : 'certificate ' + id; @@ -964,6 +1248,9 @@ _on('vv-au-panel-proxies', 'click', async e => { }); return; } + // Why + const whyBtn = e.target.closest('[data-why]'); + if (whyBtn) { _whyModal(parseInt(whyBtn.dataset.why), whyBtn); return; } // Edit const editBtn = e.target.closest('[data-proxy-edit]'); if (editBtn) { _proxyModal(parseInt(editBtn.dataset.proxyEdit)); return; } @@ -1607,8 +1894,129 @@ _on('vv-au-panel-users', 'click', async e => { } }); +// ── Access simulator ────────────────────────────────────────────────────────── +// The dropdowns come from the other two panels' data, which this panel does not otherwise load — +// AUTH_STACK decides which panels exist and _loadTab only fetches what the open one needs, so both +// lists are fetched here rather than assumed to be sitting in the page already. +function _simFill() { + const dom = document.getElementById('vv-au-sim-dom'); + const user = document.getElementById('vv-au-sim-user'); + if (!dom || !user) return; + + const fillDom = () => { + // Wildcards are dropped: they are not a hostname anyone opens, so simulating one answers a + // question nobody can ask. Disabled hosts stay in — "why does this not work" is often "it is + // switched off", and hiding them hides the answer. + const seen = new Set(); + _proxies.forEach(p => (p.domain_names || []).forEach(d => { if (!String(d).includes('*')) seen.add(String(d).toLowerCase()); })); + dom.innerHTML = [...seen].sort().map(d => ``).join('') + || ''; + }; + const fillUser = () => { + user.innerHTML = _users.slice().sort((a, b) => String(a.id).localeCompare(String(b.id))) + .map(u => ``).join('') + || ''; + }; + + if (_proxies.length) fillDom(); + else _get('npm_proxies', r => { if (r.ok) { _proxies = r.proxies || []; fillDom(); } }); + if (_users.length) fillUser(); + else _get('lldap_users', r => { if (r.ok) { _users = r.users || []; fillUser(); } }); +} + +// Held for the assistant hand-off, same as the proxy check. +let _sim = null; + +function _simRun() { + const out = document.getElementById('vv-au-sim-out'); + const btn = document.getElementById('vv-au-sim-run'); + const dom = document.getElementById('vv-au-sim-dom').value; + const uid = document.getElementById('vv-au-sim-user').value; + const path = document.getElementById('vv-au-sim-path').value || '/'; + if (!dom) return; + + _sim = null; + btn.disabled = true; + out.innerHTML = '
Reading the proxy host, the rules of the Authelia it talks to, and the directory…
'; + + _get('access_check', r => { + btn.disabled = false; + if (!r || !r.ok) { out.innerHTML = `
${_esc(r?.error || 'The check failed')}
`; return; } + _sim = r; + out.innerHTML = _simHtml(r); + const ask = document.getElementById('vv-au-sim-ask'); + if (ask) ask.onclick = () => _simAsk(r); + }, { domain: dom, uid, path }); +} + +function _simHtml(r) { + // The verdict as one word, because that is the question. Bypass is coloured as a warning rather + // than a success — reaching a page without being asked to log in is only good news if it was + // meant to be public, and the findings below say which case this is. + const pol = r.policy || 'unknown'; + const cls = pol === 'deny' ? 'bad' : (pol === 'bypass' ? 'warn' : 'ok'); + const verdict = `
${_esc(pol)}
`; + + const findings = (r.findings || []).map(f => + `
${_esc(f.text)}
`).join(''); + + // Every rule, including the ones that did nothing. Which rule you expected to win and why it was + // stepped over is the part that is impossible to see from the cards below. + const trace = (r.trace || []).map(t => + `
+ ${t.n} + ${_esc(t.label)} + ${_esc(t.why)}${t.applied ? ' → ' + _esc(t.policy || '') : ''} +
`).join(''); + + const groups = (r.groups || []).length + ? (r.groups || []).map(g => `${_esc(g)}`).join(' ') + : 'no groups'; + + const ask = document.getElementById('vv-au-ai-chat') + ? '' : ''; + + return `${verdict}${findings} +
+
Groups
${groups}
+ ${r.forward ? `
Forwards to
${_esc(r.forward)}
` : ''} + ${r.authelia && r.authelia.container ? `
Decided by
${_esc(r.authelia.container)}
${_esc(r.authelia.config || '')}
` : ''} + ${r.default_policy ? `
Default policy
${_esc(r.default_policy)}
` : ''} +
+ ${trace ? `
Every rule, in file order
${trace}` : ''} + ${ask ? `
${ask}
` : ''}`; +} + +function _simAsk(r) { + const chat = window.__vvAiChat && window.__vvAiChat['vv-au-ai']; + if (!chat) return; + if (chat.busy()) { vvAlert('The assistant is still answering the previous question.'); return; } + + const L = []; + L.push(`On this machine, can ${r.uid || 'anyone'} open https://${r.domain}${r.path}? Here is what the Auth tab worked out.`); + L.push(`The proxy host ${r.served ? 'exists' : 'does not exist'}${r.served && !r.enabled ? ' but is disabled' : ''}${r.forward ? ', forwarding to ' + r.forward : ''}.`); + L.push(`${r.uid || 'The user'} is in these LDAP groups: ${(r.groups || []).join(', ') || 'none'}.`); + if (r.authelia && r.authelia.container) + L.push(`Authentication is decided by ${r.authelia.container}, config ${r.authelia.config || 'not found'}${r.authelia.is_configured ? '' : ' — which is not the instance this tab edits'}.`); + L.push(`Rules were walked in file order:`); + for (const t of (r.trace || [])) + L.push(` ${t.n}. ${t.label} — ${t.applied ? 'APPLIED, policy ' + (t.policy || '') : 'skipped: ' + t.why}`); + L.push(`Default policy is ${r.default_policy || 'unknown'}. The resulting policy is ${r.policy || 'unknown'}.`); + L.push(''); + L.push('Is that the intended outcome? If not, say which rule to change and how — be specific about ' + + 'rule order, because Authelia stops at the first rule that matches on every axis.'); + + chat.retarget('troubleshoot', r.domain, 'now looking at access to ' + r.domain); + const input = document.getElementById('vv-au-ai-input'); + if (input) input.value = L.join('\n'); + chat.send(); + const card = document.getElementById('vv-au-ai-card'); + if (card) card.scrollIntoView({ behavior: 'smooth', block: 'nearest' }); +} + // ── Access Control ──────────────────────────────────────────────────────────── function _loadAcl() { + _simFill(); const loading = document.getElementById('vv-au-acl-loading'); const grid = document.getElementById('vv-au-acl-body'); const empty = document.getElementById('vv-au-acl-empty'); @@ -1842,6 +2250,10 @@ _on('vv-au-panel-acl', 'input', e => { // Enter adds a row underneath and moves to it, so a list of fourteen can be typed straight // through instead of returning to the add button between each one. _on('vv-au-panel-acl', 'keydown', e => { + // Enter in the path box runs the simulation. Checked before the domain-input handler below, + // which would otherwise never see it anyway but reads as if it might. + if (e.target.id === 'vv-au-sim-path' && e.key === 'Enter') { e.preventDefault(); _simRun(); return; } + const inp = e.target.closest('.vv-au-dominput'); if (!inp || e.key !== 'Enter') return; e.preventDefault(); @@ -1860,6 +2272,7 @@ function _focusDomain(r, d) { // ACL event delegation _on('vv-au-panel-acl', 'click', async e => { + if (e.target.id === 'vv-au-sim-run') { _simRun(); return; } if (e.target.id === 'vv-au-rule-add') { _ruleModal(null); return; } const domAdd = e.target.closest('[data-dom-add]'); @@ -2112,6 +2525,75 @@ _on('vv-au-cert-run', 'click', function() { _loadCerts(() => { btn.disabled = false; btn.textContent = 'Refresh'; }); }); +// ── Renewal triage ──────────────────────────────────────────────────────────── +let _triage = null; + +_on('vv-au-cert-triage', 'click', function() { + const btn = this, out = document.getElementById('vv-au-cert-triage-out'); + btn.disabled = true; + out.innerHTML = '
Reading certbot\'s logs…
'; + + fetch(CERT_API + '?action=triage').then(r => r.json()).then(r => { + btn.disabled = false; + if (!r || !r.ok) { out.innerHTML = `
${_esc(r?.error || 'Triage failed')}
`; return; } + _triage = r; + out.innerHTML = _triageHtml(r); + const ask = document.getElementById('vv-au-triage-ask'); + if (ask) ask.onclick = () => _triageAsk(r); + }).catch(e => { btn.disabled = false; out.innerHTML = `
${_esc(String(e))}
`; }); +}); + +function _triageHtml(r) { + if (!r.total) + return `
No renewal failures in the + last ${r.files_read} certbot runs. Nothing is failing to renew right now.
`; + + // Causes first and consequences after, marked as such. Sorted by count would put rate limiting + // at the top of every one of these, and rate limiting is almost never the thing to fix. + const cats = (r.categories || []).map(c => ` +
+ ${c.count} + ${_esc(c.what)}${c.root ? '' : ' — a consequence, not a cause'} + ${(c.domains || []).length ? `
${(c.domains || []).map(d => _esc(d)).join(' · ')}
` : ''} + +
`).join(''); + + const reading = (r.reading || []).map(l => `
${_esc(l)}
`).join(''); + const ask = document.getElementById('vv-au-ai-chat') + ? '
' : ''; + + return `
+
+ ${r.total} of the last ${r.files_read} certbot runs failed${r.unclassified ? ` · ${r.unclassified} matched nothing known` : ''} +
+ ${cats}${reading}${ask} +
`; +} + +function _triageAsk(r) { + const chat = window.__vvAiChat && window.__vvAiChat['vv-au-ai']; + if (!chat) return; + if (chat.busy()) { vvAlert('The assistant is still answering the previous question.'); return; } + + const L = []; + L.push(`Certificate renewals on this machine. ${r.total} of the last ${r.files_read} certbot runs failed. ` + + `Categories, counted per run rather than per log line:`); + for (const c of (r.categories || [])) + L.push(` ${c.count} — ${c.what}${c.root ? ' [root cause]' : ' [consequence]'}` + + ((c.domains || []).length ? ` — ${c.domains.join(', ')}` : '')); + for (const l of (r.reading || [])) L.push(l); + L.push(''); + L.push('What do I fix first, and in what order? Be concrete about which hostnames to add DNS for, ' + + 'delete, or remove from NPM — and say whether the rate limit clears on its own once that is done.'); + + chat.retarget('troubleshoot', 'certificates', 'now looking at certificate renewals'); + const input = document.getElementById('vv-au-ai-input'); + if (input) input.value = L.join('\n'); + chat.send(); + const card = document.getElementById('vv-au-ai-card'); + if (card) card.scrollIntoView({ behavior: 'smooth', block: 'nearest' }); +} + // ── Assistant ──────────────────────────────────────────────────────────────── // vv_ai_chat_markup() emits markup and nothing else — the instance has to be constructed here, // as every other placement does. Without this the card renders complete and is entirely inert: