bool] or an _err key — so no caller has to distinguish an exception from a // legitimately empty list. // // EXPORTS // Config vv_auth_conf(), vv_auth_creds_missing(), vv_auth_token_err(), // vv_auth_last_transport() // NPM vv_npm_list_proxies(), vv_npm_list_certs(), vv_npm_create_proxy(), // vv_npm_update_proxy(), vv_npm_delete_proxy(), vv_npm_toggle_proxy() // Diagnosis vv_npm_why(), vv_npm_why_findings(), vv_auth_uptime_window(), vv_auth_db_file(), // vv_auth_tcp_probe(), vv_auth_http_probe(), vv_auth_container_for() // — read-only. The only group here that changes nothing. // LLDAP vv_lldap_list_users(), vv_lldap_list_groups(), vv_lldap_create_user(), // vv_lldap_update_user(), vv_lldap_delete_user(), vv_lldap_set_password(), // vv_lldap_create_group(), vv_lldap_delete_group(), // vv_lldap_add_to_group(), vv_lldap_remove_from_group() // Authelia vv_authelia_read_rules(), vv_authelia_write_rules() // // CONFIGURATION // HOST*_NPM_URL admin API — port 7818. Port 81 is the partnership WebUI port // (HOST*_PARTNERSHIP_AUTH_WEBUIS), not the API. Easy to confuse. // HOST*_NPM_USER / _NPM_PASS // HOST*_LLDAP_URL / _LLDAP_USER / _LLDAP_PASS // HOST*_AUTHELIA_CONFIG path to configuration.yml. Lives in the Critical-Data share so // it is covered by the 30-minute auth sync — not under // /mnt/user/appdata, which is not synced. // HOST*_AUTHELIA_CONTAINER restarted after a successful rules write // ═══════════════════════════════════════════════════════════════════════════════════════════════ require_once __DIR__ . '/config.php'; // ── Config ──────────────────────────────────────────────────────────────────── function vv_auth_conf(): array { $v = vv_conf_vars(); $host = strtoupper(vv_detect_host()); return [ 'npm_url' => rtrim($v["{$host}_NPM_URL"] ?? 'http://localhost:7818', '/'), 'npm_user' => $v["{$host}_NPM_USER"] ?? '', 'npm_pass' => $v["{$host}_NPM_PASS"] ?? '', 'lldap_url' => rtrim($v["{$host}_LLDAP_URL"] ?? 'http://localhost:17170', '/'), 'lldap_user' => $v["{$host}_LLDAP_USER"] ?? '', 'lldap_pass' => $v["{$host}_LLDAP_PASS"] ?? '', 'authelia_config' => $v["{$host}_AUTHELIA_CONFIG"] ?? '/mnt/user/appdata-Fallback/Critical-Data/Authelia/configuration.yml', 'authelia_container' => $v["{$host}_AUTHELIA_CONTAINER"] ?? 'Authelia', 'is_owner' => vv_is_owner(), ]; } // ── Which stack ─────────────────────────────────────────────────────────────── // The stacks this page knows how to drive, and how far it can drive each one. Declared rather // than inferred so the tab can offer a stack it cannot yet operate and say exactly that, instead // of drawing panels that call endpoints with nothing behind them — which is how a switch ends up // looking like a feature while reading nothing. const VV_AUTH_STACKS = [ 'authelia_lldap' => [ 'label' => 'Authelia + lldap', 'ready' => true, 'panels' => ['proxies', 'users', 'acl', 'certs'], 'summary' => 'Authelia holds the access rules, lldap holds users and groups, ' . 'Nginx Proxy Manager holds the hostnames.', ], 'authentik' => [ 'label' => 'Authentik', 'ready' => false, // Proxies and certs are NPM's, not the identity stack's, so they keep working whichever // stack is selected. Users and access control are the two this page cannot draw yet. 'panels' => ['proxies', 'certs'], 'summary' => 'One stack for identity and access. Varaverk can still manage the proxy ' . 'hosts and certificates, but not Authentik users, groups or policies yet.', 'needs' => 'An API token and base URL in host*.conf, then user, group and policy ' . 'calls against Authentik\'s REST API to sit behind the same page.', ], ]; // Falls back rather than failing: an unrecognised value means someone typed a stack name into the // conf, and answering with a blank tab helps nobody. The working stack is the safe answer, and // vv_auth_stack_valid() is what the page uses to say the value was not understood. function vv_auth_stack(): string { $v = trim(vv_conf_vars()['AUTH_STACK'] ?? ''); return isset(VV_AUTH_STACKS[$v]) ? $v : 'authelia_lldap'; } function vv_auth_stack_valid(): bool { $v = trim(vv_conf_vars()['AUTH_STACK'] ?? ''); return $v === '' || isset(VV_AUTH_STACKS[$v]); } function vv_auth_stack_def(): array { return VV_AUTH_STACKS[vv_auth_stack()]; } // One question, asked the same way by the page and by the endpoint. The page uses it to decide // what to draw; api/auth.php uses it to refuse an action belonging to a stack that is not the one // in force, so a stale tab left open across a switch cannot write to the wrong directory. function vv_auth_panel_on(string $panel): bool { return in_array($panel, vv_auth_stack_def()['panels'], true); } // ── Credential state ────────────────────────────────────────────────────────── // Three different failures arrive at a token fetch as the same empty string: the credential was // never filled in, the service rejected it, or the service is not answering. They need three // different actions, and "check credentials" sends someone to look at a password that is fine // while the container is down — or at a container that is fine while the field is empty. // // Blank is checked first and without a request, because there is nothing to ask: a login with an // empty identity is a guess against a service that may rate-limit or lock the account, and // vv_npm_raw() would report its 401 as if a real password had been rejected. function vv_auth_creds_missing(string $svc): string { $conf = vv_auth_conf(); $h = strtoupper(vv_detect_host()); if ($svc === 'npm') return ($conf['npm_user'] === '' || $conf['npm_pass'] === '') ? "NPM credentials are not set — {$h}_NPM_USER / {$h}_NPM_PASS are empty. Fill them in Auth settings, below." : ''; return ($conf['lldap_user'] === '' || $conf['lldap_pass'] === '') ? "lldap credentials are not set — {$h}_LLDAP_USER / {$h}_LLDAP_PASS are empty. Fill them in Auth settings, below." : ''; } // The transport result of the last auth-stack curl, so a caller holding an empty token can say // which of the two remaining failures it was. Static rather than returned through every signature // because the token functions return a plain string and always have; widening them would touch // every call site to carry a value only the failure path reads. function vv_auth_last_transport(?array $set = null): array { static $last = ['errno' => 0, 'error' => '', 'code' => 0]; if ($set !== null) $last = $set; return $last; } function vv_auth_token_err(string $svc, string $url): string { $t = vv_auth_last_transport(); $name = $svc === 'npm' ? 'NPM' : 'lldap'; if ($t['errno']) return "$name unreachable at $url — " . ($t['error'] ?: 'connection failed'); $h = strtoupper(vv_detect_host()); $k = $svc === 'npm' ? "{$h}_NPM_USER / {$h}_NPM_PASS" : "{$h}_LLDAP_USER / {$h}_LLDAP_PASS"; return "$name rejected the login — check $k in Auth settings, below."; } // ── NPM ─────────────────────────────────────────────────────────────────────── function vv_npm_token(): string { if (!session_id()) session_start(); $conf = vv_auth_conf(); $cached = $_SESSION['vv_npm_token'] ?? ''; $expiry = $_SESSION['vv_npm_token_exp'] ?? 0; if ($cached && time() < $expiry) return $cached; $resp = vv_npm_raw('POST', '/api/tokens', [ 'identity' => $conf['npm_user'], 'secret' => $conf['npm_pass'], ], '', $conf); $token = $resp['token'] ?? ''; if ($token) { $_SESSION['vv_npm_token'] = $token; $_SESSION['vv_npm_token_exp'] = time() + 82800; } return $token; } function vv_npm_raw(string $method, string $path, array $data, string $token, array $conf = []): array { if (!$conf) $conf = vv_auth_conf(); $url = $conf['npm_url'] . $path; $headers = ['Content-Type: application/json', 'Accept: application/json']; if ($token) $headers[] = 'Authorization: Bearer ' . $token; $ch = curl_init($url); curl_setopt_array($ch, [ CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT => 10, CURLOPT_HTTPHEADER => $headers, CURLOPT_CUSTOMREQUEST => $method, ]); if ($data && in_array($method, ['POST', 'PUT'], true)) curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($data)); $body = curl_exec($ch); vv_auth_last_transport([ 'errno' => curl_errno($ch), 'error' => curl_error($ch), 'code' => (int) curl_getinfo($ch, CURLINFO_HTTP_CODE), ]); curl_close($ch); // NPM answers a successful DELETE with the bare JSON literal `true`, not an object. This // function is declared `: array`, so decoding that and returning it threw a TypeError and // killed the request — which is what deleting a certificate did, and what deleting or toggling // a proxy host has always done, since those three are the only callers whose endpoint answers // with a scalar. Wrapped rather than returned raw, so every caller still gets an array and the // outcome is readable as ['result' => true]. $decoded = json_decode($body ?: '{}', true); if (is_array($decoded)) return $decoded; if ($decoded === null) return []; return ['result' => $decoded]; } function vv_npm_req(string $method, string $path, array $data = []): array { if ($miss = vv_auth_creds_missing('npm')) return ['_err' => $miss]; $token = vv_npm_token(); if (!$token) return ['_err' => vv_auth_token_err('npm', vv_auth_conf()['npm_url'])]; return vv_npm_raw($method, $path, $data, $token); } function vv_npm_list_proxies(): array { $list = vv_npm_req('GET', '/api/nginx/proxy-hosts?expand=certificate'); if (!is_array($list) || isset($list['_err'])) return ['ok' => false, 'error' => $list['_err'] ?? 'Invalid response from NPM']; return ['ok' => true, 'proxies' => $list]; } // Returns a list, always. An auth failure arrives here as a map with an _err key, and the // is_array() check passed it straight through as if it were the certificates — the caller then // held an object where it expected an array and lost .find() on it. The contract is a list, so a // failure is an empty one; vv_npm_list_proxies() runs on the same page and reports the reason. function vv_npm_list_certs(): array { $list = vv_npm_req('GET', '/api/nginx/certificates'); if (!is_array($list) || isset($list['_err'])) return []; return array_values($list); } // Deleting a certificate is not undoable — the private key goes with it and a replacement means a // fresh issuance against Let's Encrypt's rate limits. The caller is expected to have checked that // no proxy host still points at it; NPM will happily remove one that is in use and leave the host // serving nothing. function vv_npm_delete_cert(int $id): array { $r = vv_npm_req('DELETE', "/api/nginx/certificates/$id"); if (is_array($r) && isset($r['_err'])) return ['ok' => false, 'error' => $r['_err']]; // NPM answers `true` for a successful delete and an error object otherwise. if (is_array($r) && isset($r['error'])) return ['ok' => false, 'error' => $r['error']['message'] ?? 'Delete failed']; return ['ok' => true]; } // Which proxy hosts reference a certificate. The guard that belongs with the delete above, so a // caller cannot forget to ask the question. function vv_npm_cert_users(int $id): array { $p = vv_npm_list_proxies(); if (!($p['ok'] ?? false)) return []; $out = []; foreach ($p['proxies'] as $h) if ((int) ($h['certificate_id'] ?? 0) === $id) $out[] = $h; return $out; } function vv_npm_create_proxy(array $data): array { $r = vv_npm_req('POST', '/api/nginx/proxy-hosts', $data); return isset($r['id']) ? ['ok' => true, 'proxy' => $r] : ['ok' => false, 'error' => $r['error'] ?? ($r['_err'] ?? 'Create failed')]; } function vv_npm_update_proxy(int $id, array $data): array { $r = vv_npm_req('PUT', "/api/nginx/proxy-hosts/$id", $data); return isset($r['id']) ? ['ok' => true, 'proxy' => $r] : ['ok' => false, 'error' => $r['error'] ?? ($r['_err'] ?? 'Update failed')]; } function vv_npm_delete_proxy(int $id): array { vv_npm_req('DELETE', "/api/nginx/proxy-hosts/$id"); return ['ok' => true]; } function vv_npm_toggle_proxy(int $id, bool $enabled): array { vv_npm_req('POST', "/api/nginx/proxy-hosts/$id/" . ($enabled ? 'enable' : 'disable')); return ['ok' => true]; } // ── Why is this host not at 100% ────────────────────────────────────────────── // The one implementation of the uptime windowing rule. Buckets are keyed by time, so "the last N" // is a key sort rather than an assumption that every period produced a sample — a pass that did not // run leaves no bucket at all rather than a zero, and averaging over a count would read a probe // outage as a service outage. Tools/uptime_probe.php and api/auth.php both defer to this. function vv_auth_uptime_window(array $buckets, int $n): ?float { if (!$buckets) return null; krsort($buckets); $u = $t = 0; foreach (array_slice($buckets, 0, $n, true) as $b) { $u += $b['u'] ?? 0; $t += $b['t'] ?? 0; } return $t > 0 ? round($u / $t * 100, 2) : null; } // The same buckets as a drawable series: one entry per period on the calendar, oldest first, null // where nothing was recorded. Aligned to the clock rather than packed, because the gap is the // information — a domain probed for two of the last thirty days must read as twenty-eight unknowns // and not as a solid bar chart that happens to be short. // // Keys are built with mktime rather than strtotime("-N month"), which resolves relative to today's // day-of-month and skips a month entirely when run on the 31st. function vv_auth_uptime_series(array $buckets, int $n, string $unit, ?int $now = null): array { $now = $now ?? time(); [$Y, $M, $D, $H] = [(int) date('Y', $now), (int) date('n', $now), (int) date('j', $now), (int) date('G', $now)]; $out = []; for ($i = $n - 1; $i >= 0; $i--) { // mktime normalises underflow, so hour -3 is 21:00 the previous day and month 0 is December. [$fmt, $ts] = match ($unit) { 'hour' => ['YmdH', mktime($H - $i, 0, 0, $M, $D, $Y)], 'month' => ['Ym', mktime(0, 0, 0, $M - $i, 1, $Y)], default => ['Ymd', mktime(0, 0, 0, $M, $D - $i, $Y)], }; $b = $buckets[date($fmt, $ts)] ?? null; $out[] = ($b && ($b['t'] ?? 0) > 0) ? round($b['u'] / $b['t'] * 100, 2) : null; } return $out; } function vv_auth_db_file(string $name): string { return rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/' . $name; } // One TCP connect, timed. The cheapest question that separates "the application is broken" from // "nothing is there at all", and the one the proxy itself cannot answer — NPM reports a 502 for a // refused connection, a closed port and a hung process alike. function vv_auth_tcp_probe(string $host, int $port, int $timeout = 4): array { if ($host === '' || $port <= 0) return ['ok' => false, 'err' => 'no forward target configured']; $t0 = microtime(true); $errno = 0; $errstr = ''; // @ because a refused connection and an unresolvable name are both expected answers here, and // a warning raised into the JSON body would corrupt the response this is reported in. $fp = @fsockopen($host, $port, $errno, $errstr, $timeout); $ms = (int) round((microtime(true) - $t0) * 1000); if ($fp === false) return ['ok' => false, 'ms' => $ms, 'err' => $errstr ?: ('errno ' . $errno)]; fclose($fp); return ['ok' => true, 'ms' => $ms]; } // A HEAD against a URL, reporting the same three things for every probe so the caller can compare // the front door and the back door without special-casing either. // // $verify is the whole point of the second call this makes: an identical request that succeeds only // with verification off says the certificate is the fault and the service behind it is fine, which // is otherwise indistinguishable from the site being down. function vv_auth_http_probe(string $url, bool $verify, int $timeout = 5): array { $ch = curl_init($url); curl_setopt_array($ch, [ CURLOPT_NOBODY => true, CURLOPT_FOLLOWLOCATION => false, // a redirect to the auth portal is an answer, not a step CURLOPT_TIMEOUT => $timeout, CURLOPT_CONNECTTIMEOUT => min($timeout, 4), // Marked as the monitor so this cannot land in the access log as a real request and inflate // the very traffic figures shown beside it. Same string npm_access_stats.php drops. CURLOPT_USERAGENT => 'Varaverk-Uptime/1.0', CURLOPT_RETURNTRANSFER => true, CURLOPT_SSL_VERIFYPEER => $verify, CURLOPT_SSL_VERIFYHOST => $verify ? 2 : 0, ]); curl_exec($ch); $r = ['code' => (int) curl_getinfo($ch, CURLINFO_HTTP_CODE), 'ms' => (int) round(curl_getinfo($ch, CURLINFO_TOTAL_TIME) * 1000), 'err' => curl_error($ch)]; curl_close($ch); return $r; } // Which container is behind a forward target, if any. Matched on the name first and the container // IP second, because both forms are in use here — some hosts forward to a container name on a // custom network and some to an address on br0. // // Returns null rather than guessing. A forward target that is another machine entirely is a normal // configuration, and reporting "no container" for it is correct, not a failure to find one. function vv_auth_container_for(string $fwdHost): ?array { if ($fwdHost === '') return null; require_once __DIR__ . '/docker.php'; $all = vv_dk_inspect_all(); if (!$all) return null; $hit = function (string $name, array $c, string $how): array { $nets = $c['networks'] ?? []; return ['name' => $name, 'running' => $c['running'], 'status' => $c['status'], 'match' => $how, 'networks' => $nets, 'ip' => $nets ? reset($nets) : '', 'network' => $nets ? (string) array_key_first($nets) : '']; }; $needle = strtolower($fwdHost); foreach ($all as $name => $c) if (strtolower($name) === $needle) return $hit($name, $c, 'name'); foreach ($all as $name => $c) foreach ($c['networks'] ?? [] as $ip) if ($ip === $fwdHost) return $hit($name, $c, 'ip'); return null; } // Where this process should actually knock, which is not always what the proxy host says. // // Most forward targets here are container names on a user-defined docker network. Those names are // resolved by docker's embedded DNS, which only the containers on that network can see — NPM // resolves NextCloud perfectly and PHP running on the host cannot resolve it at all. A check that // treated its own resolution failure as evidence would report every one of them as dead, which is // the exact false alarm this whole dialog exists to stop someone chasing. // // So an unresolvable name that matches a running container is redirected to that container's // address, and the substitution is reported rather than hidden — the reader needs to know which // address the result below actually describes. function vv_auth_probe_target(string $fwdHost, ?array $container): array { if ($fwdHost === '' || filter_var($fwdHost, FILTER_VALIDATE_IP)) return ['host' => $fwdHost, 'note' => '']; // gethostbyname() hands back its input unchanged when it cannot resolve — the documented way it // fails, and the reason this is a comparison rather than a truthiness test. if (@gethostbyname($fwdHost) !== $fwdHost) return ['host' => $fwdHost, 'note' => '']; if ($container && ($container['ip'] ?? '') !== '') return ['host' => $container['ip'], 'note' => $fwdHost . ' is a docker name that only resolves on the ' . ($container['network'] ?: 'proxy') . ' network, so this checked the container address ' . $container['ip'] . ' instead.']; return ['host' => $fwdHost, 'note' => $fwdHost . ' does not resolve from this host and matches no container here, so the direct ' . 'check below could not be made. What the proxy itself reports is the reliable part.']; } // Everything known about why one proxy host is not at 100%, gathered in one pass. // // The Proxies tab shows an uptime percentage and nothing about what is behind it, and the causes // look identical from the row: the application is down, its container is not running, the // certificate stopped validating, or the proxy reaches the application perfectly and the // application is the thing returning 5xx. Those are four different jobs and the row cannot tell // them apart, so every low figure has so far meant opening NPM, then Docker, then a terminal. // // Three of the four sources are already on disk — the probe history, the access-log totals, NPM's // own record. The fourth is the part no stored figure can answer: what happens right now, asked // separately of the front door and the back. A host that answers on 10.0.0.5:8096 but fails through // https://name/ is a proxy or certificate fault; one that fails both is the service itself. // // Read-only by construction. Every call below is a GET, a HEAD, a TCP connect or a file read — // nothing here restarts, rewrites or retries anything, because the value of a diagnosis is that it // can be run on a host that is limping without being the thing that finishes it off. function vv_npm_why(int $id): array { $p = vv_npm_list_proxies(); if (!($p['ok'] ?? false)) return ['ok' => false, 'error' => $p['error'] ?? 'Could not read proxy hosts from NPM']; $host = null; foreach ($p['proxies'] as $h) if ((int) ($h['id'] ?? 0) === $id) { $host = $h; break; } if (!$host) return ['ok' => false, 'error' => "No proxy host with id $id — the list may be stale, reload the tab."]; $adv = trim((string) ($host['advanced_config'] ?? '')); $guarded = str_contains($adv, 'auth_request'); $fwdHost = trim((string) ($host['forward_host'] ?? '')); $fwdPort = (int) ($host['forward_port'] ?? 0); $fwdScheme = (string) ($host['forward_scheme'] ?? 'http'); $enabled = ($host['enabled'] ?? true) ? true : false; // Wildcards are excluded for the same reason the probe excludes them: *.example.com is not a // hostname anything can connect to, so a live check against it would report a fault that only // describes the check. $domains = []; foreach ($host['domain_names'] ?? [] as $d) { $d = strtolower(trim((string) $d)); if ($d !== '' && !str_contains($d, '*')) $domains[] = $d; } $out = [ 'ok' => true, 'id' => $id, 'enabled' => $enabled, 'domains' => $host['domain_names'] ?? [], 'forward' => $fwdScheme . '://' . $fwdHost . ':' . $fwdPort, 'guarded' => $guarded, 'checked' => time(), ]; // ── Certificate ── // Expiry is the single most common reason a host that worked for months stops, and the row // cannot show it because the row is about uptime. $cert = null; $certId = (int) ($host['certificate_id'] ?? 0); if ($certId > 0) { foreach (vv_npm_list_certs() as $c) { if ((int) ($c['id'] ?? 0) !== $certId) continue; $exp = strtotime((string) ($c['expires_on'] ?? '')) ?: null; $cert = [ 'name' => $c['nice_name'] ?? implode(', ', $c['domain_names'] ?? []), 'provider' => $c['provider'] ?? '', 'expires' => $exp, 'days_left' => $exp ? (int) floor(($exp - time()) / 86400) : null, ]; break; } } $out['cert'] = $cert; // ── Recorded history ── $u = is_file(vv_auth_db_file('uptime.json')) ? (json_decode((string) @file_get_contents(vv_auth_db_file('uptime.json')), true) ?: []) : []; $recs = []; foreach ($domains as $d) { $r = $u['domains'][$d] ?? null; if (!$r) continue; // The hours that actually lost something, rather than all 48. "Every hour lost two samples" // and "one hour lost forty" are the same daily percentage and completely different faults, // and this is the only place that distinction survives. $bad = []; $hours = $r['hours'] ?? []; krsort($hours); foreach (array_slice($hours, 0, 24, true) as $k => $b) { $t = $b['t'] ?? 0; $up = $b['u'] ?? 0; if ($t > 0 && $up < $t) $bad[] = ['hour' => $k, 'up' => $up, 'total' => $t]; } $recs[$d] = [ 'state' => $r['state'] ?? null, 'h1' => vv_auth_uptime_window($r['hours'] ?? [], 1), 'h24' => vv_auth_uptime_window($r['hours'] ?? [], 24), 'd30' => vv_auth_uptime_window($r['days'] ?? [], 30), 'last_code' => $r['last_code'] ?? null, 'last_detail' => $r['last_detail'] ?? null, 'last_change' => $r['last_change'] ?? null, 'last_ms' => $r['last_ms'] ?? null, 'checks' => $r['checks'] ?? 0, // Newest first — a flap is read backwards from now, not forwards from whenever the // record happens to start. 'events' => array_slice(array_reverse($r['events'] ?? []), 0, 8), 'bad_hours' => $bad, ]; } $out['history'] = $recs; // ── Access-log totals ── $a = is_file(vv_auth_db_file('npm_access.json')) ? (json_decode((string) @file_get_contents(vv_auth_db_file('npm_access.json')), true) ?: []) : []; $out['traffic'] = $a['hosts'][(string) $id] ?? null; // ── Live, right now ── $cont = vv_auth_container_for($fwdHost); $target = vv_auth_probe_target($fwdHost, $cont); $out['upstream'] = [ 'host' => $fwdHost, 'port' => $fwdPort, 'probed' => $target['host'], 'note' => $target['note'], 'container' => $cont, 'tcp' => vv_auth_tcp_probe($target['host'], $fwdPort), ]; // Only worth asking once something is listening — an HTTP probe of a closed port re-reports the // TCP failure in a less specific form. if ($out['upstream']['tcp']['ok'] ?? false) { // Verification off deliberately: this is an internal hop to an address on this machine's own // network, usually plain HTTP and usually a self-signed certificate when it is not. The // question here is whether the application answers, and the certificate question is asked // at the front door where it actually applies. $out['upstream']['http'] = vv_auth_http_probe($fwdScheme . '://' . $target['host'] . ':' . $fwdPort . '/', false); } // Bounded. A host carrying a dozen names would otherwise turn one button press into a dozen // sequential TLS handshakes, and the first few answer the question. $out['live'] = []; foreach (array_slice($domains, 0, 4) as $d) { $r = vv_auth_http_probe('https://' . $d . '/', true); // The second call is the diagnosis, not a retry: succeeding here after failing above is // what proves the certificate rather than the service. if ($r['err'] !== '' && preg_match('/certificat|SSL|TLS/i', $r['err'])) { $r['insecure'] = vv_auth_http_probe('https://' . $d . '/', false); } $out['live'][$d] = $r; } $out['findings'] = vv_npm_why_findings($out); return $out; } // The deterministic half of the answer, written as sentences rather than codes. // // Separate from the gathering so it can be read, argued with and corrected on its own — and so the // dialog has something certain to show whether or not there is a model on this node to interpret // it. Most low figures on this installation have one of these causes, and none of them needs a // language model to reach. // // Ordered most decisive first: the caller shows them in order and the first line is meant to be the // answer. Each entry is ['level' => bad|warn|info, 'text' => …]. function vv_npm_why_findings(array $w): array { $f = []; $up = $w['upstream'] ?? []; $tcp = $up['tcp'] ?? []; $cont = $up['container'] ?? null; // Whether the front door is failing *now*, decided once. Several readings below change meaning // entirely on it — "the application answers directly" is a useful clue during an outage and a // false alarm when the site is simply working, and an earlier draft said the second as if it // were the first. $frontBad = false; foreach ($w['live'] ?? [] as $r) if (($r['code'] ?? 0) === 0 || ($r['code'] ?? 0) >= 500) $frontBad = true; if (!($w['enabled'] ?? true)) $f[] = ['level' => 'bad', 'text' => 'This host is disabled in NPM, so nothing is being served for it. The probe still counts it as unreachable.']; if ($cont && !($cont['running'] ?? true)) $f[] = ['level' => 'bad', 'text' => 'Container ' . $cont['name'] . ' is ' . ($cont['status'] ?? 'not running') . ' — nothing can answer on ' . ($up['host'] ?? '') . ':' . ($up['port'] ?? '') . '.']; // The direct check, stated as what it is. Whether it could be made at all is reported first, // because a check that did not happen must never be read as a check that failed — see // vv_auth_probe_target(). Most forward targets on this machine are docker names this process // cannot resolve, and an earlier draft of this reported every one of them as a dead service. $reached = ($up['probed'] ?? '') !== '' && (($tcp['ok'] ?? false) || !str_contains((string) ($tcp['err'] ?? ''), 'getaddrinfo')); if (!$reached) { $f[] = ['level' => 'info', 'text' => ($up['note'] ?: 'The forward target could not be resolved from here, so no direct check was made.') . ' Nothing below is evidence that the service is down.']; } elseif (!($tcp['ok'] ?? false)) { $f[] = ['level' => 'bad', 'text' => 'Nothing is listening on ' . ($up['probed'] ?? '') . ':' . ($up['port'] ?? '') . ' — ' . ($tcp['err'] ?? 'no reason given') . ($cont && ($cont['running'] ?? false) ? '. Container ' . $cont['name'] . ' is running, so the container is up and the application inside it is not serving that port.' : '. The proxy has nothing to forward to.')]; } else { $uh = $up['http'] ?? null; if ($uh && $uh['code'] >= 500) $f[] = ['level' => 'bad', 'text' => 'The service is listening on ' . ($up['probed'] ?? '') . ':' . ($up['port'] ?? '') . ' and answered HTTP ' . $uh['code'] . ' itself. The proxy is forwarding correctly — the fault is inside the application.']; elseif ($uh && $uh['code'] === 0 && ($uh['err'] ?? '') !== '') $f[] = ['level' => 'warn', 'text' => 'The port on ' . ($up['probed'] ?? '') . ' is open but nothing came back over it (' . $uh['err'] . '). Something is holding the socket without serving — which is what the proxy sees as a timeout.']; elseif ($uh && $frontBad) $f[] = ['level' => 'info', 'text' => 'The service answers directly on ' . ($up['probed'] ?? '') . ':' . ($up['port'] ?? '') . ' with HTTP ' . $uh['code'] . ', so whatever is failing sits between the proxy and it, not in the application.']; } // The certificate, from both directions: what NPM says about its expiry, and what a live // handshake actually did. Either can be the fault on its own — a cert with weeks left still // fails if the chain it is serving is wrong. $c = $w['cert'] ?? null; if ($c && $c['days_left'] !== null) { if ($c['days_left'] < 0) $f[] = ['level' => 'bad', 'text' => 'The certificate expired ' . abs($c['days_left']) . ' days ago. Every HTTPS request to this host fails verification.']; elseif ($c['days_left'] <= 14) $f[] = ['level' => 'warn', 'text' => 'The certificate expires in ' . $c['days_left'] . ' days — check the Certs tab for whether renewal is running.']; } foreach ($w['live'] ?? [] as $dom => $r) { if (isset($r['insecure']) && $r['insecure']['code'] > 0 && $r['insecure']['code'] < 500) { $f[] = ['level' => 'bad', 'text' => $dom . ' answers normally when certificate verification is turned off. ' . 'The service is up and the certificate is what is failing: ' . $r['err']]; } elseif ($r['code'] === 0 && ($r['err'] ?? '') !== '') { $f[] = ['level' => 'bad', 'text' => $dom . ' did not answer just now — ' . $r['err']]; } elseif ($r['code'] === 502 || $r['code'] === 504) { // NPM's own verdict, and the one piece of evidence that is always authoritative: it is // the component that actually has to reach the upstream. Paired with a direct probe // that succeeded, it stops being "the app is down" and becomes a routing problem — // the proxy and the application are on networks that cannot see each other. $ok = ($up['http']['code'] ?? 0) > 0 && ($up['http']['code'] ?? 0) < 500; $f[] = ['level' => 'bad', 'text' => $dom . ' answered HTTP ' . $r['code'] . ' through the proxy — NPM could not reach ' . ($up['host'] ?? '') . ':' . ($up['port'] ?? '') . '.' . ($ok ? ' It answers fine when asked directly, so the two are not on a network that can see each other,' . ' or the forward host is written in a form NPM cannot resolve.' : '')]; } elseif ($r['code'] >= 500) { $f[] = ['level' => 'bad', 'text' => $dom . ' answered HTTP ' . $r['code'] . ' through the proxy.']; } } // Traffic, read against whether the host is guarded. An unguarded host that is nothing but 4xx // is broken; a guarded one that is nothing but 4xx is usually Authelia doing its job to // unauthenticated callers, and calling that a fault would send someone to fix what is working. $t = $w['traffic'] ?? null; if ($t && ($t['requests'] ?? 0) > 0) { $req = (int) $t['requests']; $s5 = (int) ($t['s5xx'] ?? 0); $s4 = (int) ($t['s4xx'] ?? 0); if ($s5 / $req >= 0.5) $f[] = ['level' => 'bad', 'text' => round($s5 / $req * 100) . '% of all logged requests to this host are 5xx — this has been failing for real users, not just the probe.']; if ($s4 / $req >= 0.9) $f[] = $w['guarded'] ? ['level' => 'info', 'text' => round($s4 / $req * 100) . '% of requests are 4xx, which on a host behind auth_request is usually Authelia refusing unauthenticated callers rather than a fault.'] : ['level' => 'warn', 'text' => round($s4 / $req * 100) . '% of requests are 4xx and nothing is guarding this host, so callers are being refused by the application itself.']; } // Shape of the loss. Same percentage, two entirely different problems, and this is the only // reading that separates them. foreach ($w['history'] ?? [] as $dom => $h) { $ev = $h['events'] ?? []; // Over what period, not just how many. Four state changes is flapping if they were this // afternoon and completely unremarkable if they were spread across a month, and a count on // its own cannot tell those apart — the store keeps the last twenty however old they are. $span = count($ev) >= 2 ? ((int) ($ev[0]['ts'] ?? 0) - (int) ($ev[count($ev) - 1]['ts'] ?? 0)) : 0; $spanTxt = $span >= 172800 ? 'over ' . round($span / 86400) . ' days' : ($span >= 7200 ? 'over ' . round($span / 3600) . ' hours' : 'within the hour'); if (count($ev) >= 4) $f[] = ['level' => 'warn', 'text' => $dom . ' changed state ' . count($ev) . ' times ' . $spanTxt . ' — this is flapping rather than one clean outage, so look for something restarting on a cycle.']; elseif (count($h['bad_hours'] ?? []) === 1 && ($h['state'] ?? '') === 'up') $f[] = ['level' => 'info', 'text' => $dom . ' lost samples in one hour only (' . $h['bad_hours'][0]['up'] . '/' . $h['bad_hours'][0]['total'] . ' at ' . substr((string) $h['bad_hours'][0]['hour'], 8, 2) . ':00) and has been up since. A single event, already over.']; } if (!$f) $f[] = ['level' => 'info', 'text' => 'Nothing is failing right now — the service answers, the certificate verifies and the proxy is forwarding. Whatever cost this host its uptime is in the history below and has already ended.']; return $f; } // ── lldap ───────────────────────────────────────────────────────────────────── function vv_lldap_token(): string { if (!session_id()) session_start(); $conf = vv_auth_conf(); $cached = $_SESSION['vv_lldap_token'] ?? ''; $expiry = $_SESSION['vv_lldap_token_exp'] ?? 0; if ($cached && time() < $expiry) return $cached; $ch = curl_init($conf['lldap_url'] . '/auth/simple/login'); curl_setopt_array($ch, [ CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT => 10, CURLOPT_POST => true, CURLOPT_POSTFIELDS => json_encode(['username' => $conf['lldap_user'], 'password' => $conf['lldap_pass']]), CURLOPT_HTTPHEADER => ['Content-Type: application/json'], ]); $body = curl_exec($ch); vv_auth_last_transport([ 'errno' => curl_errno($ch), 'error' => curl_error($ch), 'code' => (int) curl_getinfo($ch, CURLINFO_HTTP_CODE), ]); curl_close($ch); $resp = json_decode($body ?: '{}', true) ?: []; $token = $resp['token'] ?? ''; if ($token) { $_SESSION['vv_lldap_token'] = $token; $_SESSION['vv_lldap_token_exp'] = time() + 3500; } return $token; } function vv_lldap_gql(string $query, array $variables = []): array { $conf = vv_auth_conf(); if ($miss = vv_auth_creds_missing('lldap')) return ['errors' => [['message' => $miss]]]; $token = vv_lldap_token(); if (!$token) return ['errors' => [['message' => vv_auth_token_err('lldap', $conf['lldap_url'])]]]; $ch = curl_init($conf['lldap_url'] . '/api/graphql'); curl_setopt_array($ch, [ CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT => 10, CURLOPT_POST => true, CURLOPT_POSTFIELDS => json_encode(['query' => $query, 'variables' => $variables]), CURLOPT_HTTPHEADER => ['Content-Type: application/json', 'Authorization: Bearer ' . $token], ]); $body = curl_exec($ch); curl_close($ch); return json_decode($body ?: '{}', true) ?: []; } function vv_lldap_list_users(): array { // firstName/lastName/uuid were never requested, so the page could not show or edit them — // 31 of the 33 users here have them set and none of it was reachable without opening lldap's // own WebUI. // // The avatar itself is deliberately NOT in this query. It is a base64 JPEG stored inline, and // the six that exist here come to 470 KB — a third of a megabyte added to every load of the // tab, re-fetched on every refresh, to draw six thumbnails. The attribute *names* are enough // to know who has one, and the bytes are fetched per user by vv_lldap_avatar() through an // endpoint the browser can cache like any other image. $r = vv_lldap_gql('query { users { id displayName email firstName lastName uuid creationDate ' . 'groups { id displayName } attributes { name } } }'); if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Query failed']; $users = $r['data']['users'] ?? []; foreach ($users as &$u) { $names = array_column($u['attributes'] ?? [], 'name'); $u['has_avatar'] = in_array('avatar', $names, true); // Sent to the browser as a flag, not a list. Nothing on the page reads the attribute names // and shipping 33 copies of the same nine strings is pure weight. unset($u['attributes']); } unset($u); return ['ok' => true, 'users' => $users]; } // Raw JPEG bytes for one user, or '' when they have no avatar. Returned as bytes rather than // base64 because the only caller streams it to an , and re-encoding it to hand the browser // something it would immediately decode again is a third of a megabyte of nothing. function vv_lldap_avatar(string $userId): string { $r = vv_lldap_gql('query Avatar($id: String!) { user(userId: $id) { avatar } }', ['id' => $userId]); if (isset($r['errors'])) return ''; $b64 = $r['data']['user']['avatar'] ?? ''; if (!is_string($b64) || $b64 === '') return ''; $raw = base64_decode($b64, true); return ($raw !== false && vv_lldap_is_jpeg($raw)) ? $raw : ''; } // lldap types this attribute JPEG_PHOTO and rejects anything else, so the check happens here where // the answer can name the problem. A rejection from the server arrives as a generic GraphQL error // several layers from the file the operator picked. function vv_lldap_is_jpeg(string $raw): bool { return strlen($raw) > 3 && substr($raw, 0, 3) === "\xFF\xD8\xFF"; } // One megabyte of JPEG, decoded. The browser resizes before upload so nothing near this should // arrive; the cap is here because this value is stored inline in the directory and read back on // every user query, and an unbounded one would be paid for on every page load forever. const VV_LLDAP_AVATAR_MAX = 1048576; function vv_lldap_set_avatar(string $userId, string $b64): array { $b64 = preg_replace('#^data:image/[a-z+]+;base64,#i', '', trim($b64)); $raw = base64_decode($b64, true); if ($raw === false || $raw === '') return ['ok' => false, 'error' => 'Image data could not be decoded']; if (!vv_lldap_is_jpeg($raw)) return ['ok' => false, 'error' => 'lldap stores avatars as JPEG only — that file is not one']; if (strlen($raw) > VV_LLDAP_AVATAR_MAX) return ['ok' => false, 'error' => 'Image is ' . round(strlen($raw) / 1024) . ' KB; the limit is ' . round(VV_LLDAP_AVATAR_MAX / 1024) . ' KB']; $r = vv_lldap_gql('mutation SetAvatar($user: UpdateUserInput!) { updateUser(user: $user) { ok } }', ['user' => ['id' => $userId, 'avatar' => base64_encode($raw)]]); if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Avatar update failed']; return ['ok' => true, 'bytes' => strlen($raw)]; } // Cleared through removeAttributes rather than by setting avatar to an empty string: lldap treats // an empty avatar as a value to validate, and it is not a JPEG. function vv_lldap_remove_avatar(string $userId): array { $r = vv_lldap_gql('mutation ClearAvatar($user: UpdateUserInput!) { updateUser(user: $user) { ok } }', ['user' => ['id' => $userId, 'removeAttributes' => ['avatar']]]); if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Avatar removal failed']; return ['ok' => true]; } function vv_lldap_list_groups(): array { $r = vv_lldap_gql('query { groups { id displayName users { id displayName } } }'); if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Query failed']; return ['ok' => true, 'groups' => $r['data']['groups'] ?? []]; } function vv_lldap_create_user(string $id, string $email, string $displayName, string $password, string $firstName = '', string $lastName = ''): array { $user = ['id' => $id, 'email' => $email, 'displayName' => $displayName]; // Omitted when blank rather than sent as "". lldap distinguishes the two, and an empty string // creates the attribute holding nothing, which then shows as set everywhere that tests for it. if ($firstName !== '') $user['firstName'] = $firstName; if ($lastName !== '') $user['lastName'] = $lastName; $r = vv_lldap_gql( 'mutation CreateUser($user: CreateUserInput!) { createUser(user: $user) { id displayName email } }', ['user' => $user] ); if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Create failed']; if ($password) vv_lldap_set_password($id, $password); return ['ok' => true, 'user' => $r['data']['createUser'] ?? []]; } // $firstName/$lastName are nullable on purpose: null means "the form did not offer this field, so // leave it alone", '' means "the operator cleared it". Passing '' for an absent field would erase // a name that 31 of the 33 users here have set. function vv_lldap_update_user(string $id, string $email, string $displayName, ?string $firstName = null, ?string $lastName = null): array { $user = ['id' => $id, 'email' => $email, 'displayName' => $displayName]; $remove = []; foreach (['firstName' => $firstName, 'lastName' => $lastName] as $k => $v) { if ($v === null) continue; if ($v === '') $remove[] = $k === 'firstName' ? 'first_name' : 'last_name'; else $user[$k] = $v; } // Clearing goes through removeAttributes — setting the field to "" leaves the attribute in // place holding an empty string, which is a different thing to lldap and to anything reading // the directory over LDAP. if ($remove) $user['removeAttributes'] = $remove; $r = vv_lldap_gql( 'mutation UpdateUser($user: UpdateUserInput!) { updateUser(user: $user) { ok } }', ['user' => $user] ); if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Update failed']; return ['ok' => true]; } function vv_lldap_delete_user(string $id): array { $r = vv_lldap_gql( 'mutation DeleteUser($userId: String!) { deleteUser(userId: $userId) { ok } }', ['userId' => $id] ); if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Delete failed']; return ['ok' => true]; } function vv_lldap_set_password(string $userId, string $password): array { $conf = vv_auth_conf(); if ($miss = vv_auth_creds_missing('lldap')) return ['ok' => false, 'error' => $miss]; $token = vv_lldap_token(); if (!$token) return ['ok' => false, 'error' => vv_auth_token_err('lldap', $conf['lldap_url'])]; $ch = curl_init($conf['lldap_url'] . '/auth/admin/resetPassword'); curl_setopt_array($ch, [ CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT => 10, CURLOPT_POST => true, CURLOPT_POSTFIELDS => json_encode(['userId' => $userId, 'password' => $password]), CURLOPT_HTTPHEADER => ['Content-Type: application/json', 'Authorization: Bearer ' . $token], ]); $body = curl_exec($ch); $code = curl_getinfo($ch, CURLINFO_HTTP_CODE); curl_close($ch); if ($code >= 200 && $code < 300) return ['ok' => true]; $err = json_decode($body ?: '{}', true)['message'] ?? "HTTP $code"; return ['ok' => false, 'error' => $err]; } function vv_lldap_create_group(string $name): array { $r = vv_lldap_gql( 'mutation CreateGroup($name: String!) { createGroup(name: $name) { id displayName } }', ['name' => $name] ); if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Create failed']; return ['ok' => true, 'group' => $r['data']['createGroup'] ?? []]; } // The only editable field a group has. Without it the sole way to correct a group's name was to // delete it and make a new one — which drops every member, and on this directory those group names // are what the Authelia rules match on, so the rule would keep naming a group that no longer // exists and quietly stop admitting anyone. function vv_lldap_rename_group(int $id, string $displayName): array { $r = vv_lldap_gql( 'mutation UpdateGroup($group: UpdateGroupInput!) { updateGroup(group: $group) { ok } }', ['group' => ['id' => $id, 'displayName' => $displayName]] ); if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Rename failed']; return ['ok' => true]; } function vv_lldap_delete_group(int $id): array { $r = vv_lldap_gql( 'mutation DeleteGroup($groupId: Int!) { deleteGroup(groupId: $groupId) { ok } }', ['groupId' => $id] ); if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Delete failed']; return ['ok' => true]; } function vv_lldap_add_to_group(string $userId, int $groupId): array { $r = vv_lldap_gql( 'mutation AddUserToGroup($userId: String!, $groupId: Int!) { addUserToGroup(userId: $userId, groupId: $groupId) { ok } }', ['userId' => $userId, 'groupId' => $groupId] ); if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Failed']; return ['ok' => true]; } function vv_lldap_remove_from_group(string $userId, int $groupId): array { $r = vv_lldap_gql( 'mutation RemoveUserFromGroup($userId: String!, $groupId: Int!) { removeUserFromGroup(userId: $userId, groupId: $groupId) { ok } }', ['userId' => $userId, 'groupId' => $groupId] ); if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Failed']; return ['ok' => true]; } // ── Authelia ────────────────────────────────────────────────────────────────── // $file overrides the configured path. There is more than one Authelia on this machine — the // primary serves the .com names and Authelia-Secondary serves the .us ones — and conf names only // the primary, so anything reasoning about a specific domain has to be able to read the instance // that domain actually talks to. Defaults to the configured one, so every existing caller and the // whole editing path are unchanged. function vv_authelia_read_rules(?string $file = null): array { $conf = vv_auth_conf(); $file = $file ?: $conf['authelia_config']; if (!file_exists($file)) return ['ok' => false, 'error' => 'Config not found: ' . $file]; $content = file_get_contents($file); if ($content === false) return ['ok' => false, 'error' => 'Cannot read config file']; // Extract default_policy, and keep any trailing comment rather than dropping it. The line in // this config reads "default_policy: bypass #deny" — a note about what it used to be, or is // meant to become, on the single most consequential setting in the file. The block is // re-emitted on save, so anything not carried here is deleted by the next save. $defaultPolicy = 'deny'; $defaultNote = ''; if (preg_match('/^[ \t]+default_policy:[ \t]+([a-z_]+)[ \t]*(#[^\n]*)?/m', $content, $m)) { $defaultPolicy = $m[1]; $defaultNote = trim($m[2] ?? ''); } // Extract the indented block under access_control: if (!preg_match('/^access_control:[ \t]*\n((?:[ \t][^\n]*\n?)*)/m', $content, $m)) return ['ok' => false, 'error' => 'access_control section not found']; $acBlock = $m[1]; // Extract the indented block under rules: (3+ space indent = rule list items) if (!preg_match('/^ rules:[ \t]*\n((?:[ \t]{3,}[^\n]*\n?)*)/m', $acBlock, $m)) return ['ok' => true, 'default_policy' => $defaultPolicy, 'rules' => [], 'default_note' => $defaultNote]; // Walked rather than preg_split, so the comment lines above each rule can be attached to it. // // This block is rebuilt from the parsed model on every save, so anything the parser drops is // deleted the next time anyone touches this page — and the parser dropped every comment. The // five rules here are labelled ## Media_Users_users, ## Admin Only, ## super_users, // ## power_users and ## Home_users, which is the only thing in the file that says what a rule // is *for*: the rule itself is thirteen hostnames and a group id. Saving once erased all five. // // A lookahead split cannot do this, because the comment above rule N lands at the end of rule // N-1's chunk (or before the first chunk entirely), so it would be attributed to the wrong // rule or lost with the preamble. $lines = explode("\n", $m[1]); $starts = []; foreach ($lines as $i => $l) if (preg_match('/^ - /', $l)) $starts[] = $i; $rules = []; foreach ($starts as $n => $s) { // Contiguous comment lines immediately above this rule, in file order. A blank line or // any content ends the run — a comment separated from the rule by a blank belongs to the // block, not to the rule. $label = []; for ($j = $s - 1; $j >= 0; $j--) { if (!preg_match('/^\s*#/', $lines[$j])) break; array_unshift($label, trim($lines[$j])); } $end = $starts[$n + 1] ?? count($lines); $chunk = implode("\n", array_slice($lines, $s, $end - $s)); $rule = vv_authelia_parse_rule_chunk($chunk); if (empty($rule)) continue; // Underscore-prefixed so it cannot collide with an Authelia field name, and so the writer // can tell presentation from configuration when it decides what to emit as YAML. if ($label) $rule['_label'] = $label; $rules[] = $rule; } return ['ok' => true, 'default_policy' => $defaultPolicy, 'rules' => $rules, 'default_note' => $defaultNote]; } function vv_authelia_parse_rule_chunk(string $chunk): array { $rule = []; $field = null; $list = []; $save = function () use (&$rule, &$field, &$list) { if ($field === null) return; if (!empty($list)) $rule[$field] = count($list) === 1 ? $list[0] : $list; $field = null; $list = []; }; foreach (explode("\n", $chunk) as $line) { $raw = rtrim($line); $trim = trim($raw); if ($trim === '' || preg_match('/^#+/', $trim)) continue; $indent = strlen($raw) - strlen(ltrim($raw, ' ')); // indent=4, starts with "- " → first field of this rule block if ($indent === 4 && str_starts_with($trim, '- ')) { $rest = ltrim(substr($trim, 2)); if (preg_match('/^([a-z_]+):[ \t]*(.*)$/', $rest, $m)) { $save(); $field = $m[1]; $val = trim($m[2]); if ($val !== '' && !str_starts_with($val, '#')) { $rule[$field] = vv_authelia_unquote($val); $field = null; } } continue; } // indent=6 → named field (scalar or list header) if ($indent === 6 && preg_match('/^([a-z_]+):[ \t]*(.*)$/', $trim, $m)) { $save(); $field = $m[1]; $val = trim($m[2]); if ($val !== '' && !str_starts_with($val, '#')) { $rule[$field] = vv_authelia_unquote($val); $field = null; } continue; } // indent=8, starts with "- " → list item under current field if ($indent === 8 && str_starts_with($trim, '- ')) { $list[] = vv_authelia_parse_list_item(trim(substr($trim, 2))); } } $save(); return $rule; } // Strip surrounding quotes and inline comments from a YAML scalar. function vv_authelia_unquote(string $val): string { $val = trim($val); $val = preg_replace('/\s+#[^"\']*$/', '', $val); // strip trailing comment if (preg_match('/^(["\'])(.+)\1$/', $val, $m)) return $m[2]; return $val; } // Parse a YAML list item: flow sequence ['group:name'] or plain/quoted scalar. function vv_authelia_parse_list_item(string $val): string { $val = trim($val); // Flow sequence: ['value'] or ["value"] or [value] if (preg_match('/^\[[\'""]?([^\]\'""]+)[\'""]?\]$/', $val, $m)) return trim($m[1]); return vv_authelia_unquote($val); } function vv_authelia_write_rules(array $rules, string $defaultPolicy, string $defaultNote = ''): array { $conf = vv_auth_conf(); $file = $conf['authelia_config']; if (!file_exists($file)) return ['ok' => false, 'error' => 'Config not found: ' . $file]; $content = file_get_contents($file); if ($content === false) return ['ok' => false, 'error' => 'Cannot read config file']; // Build the new access_control block $block = "access_control:\n"; $block .= " default_policy: $defaultPolicy" . ($defaultNote !== '' ? ' ' . $defaultNote : '') . "\n"; $block .= " rules:\n"; // Preferred field output order $fieldOrder = ['domain', 'policy', 'subject', 'networks', 'resources']; foreach ($rules as $rule) { // The labels the operator wrote above this rule, put back before it. Emitted here rather // than inside the field loop because they are not a field — they carry no indent-4 dash // and must land above the rule, not inside it. foreach ((array) ($rule['_label'] ?? []) as $lbl) { $lbl = trim((string) $lbl); if ($lbl === '') continue; // Forced back into comment form. This string reaches here from the browser, and a // label that lost its # would be spliced into the config as YAML. if ($lbl[0] !== '#') $lbl = '# ' . $lbl; // One line only — a newline here would end the comment and start config. $block .= ' ' . str_replace(["\r", "\n"], ' ', $lbl) . "\n"; } $keys = array_merge( array_filter($fieldOrder, fn($k) => array_key_exists($k, $rule)), array_diff(array_keys($rule), $fieldOrder) ); // Presentation, already emitted above. Left in the key list it would be written out as a // YAML field named _label, which Authelia would reject on load. $keys = array_filter($keys, fn($k) => $k !== '_label'); $first = true; foreach ($keys as $key) { if (!array_key_exists($key, $rule)) continue; $val = $rule[$key]; $prefix = $first ? ' - ' : ' '; $first = false; // domain, subject, resources, networks → always output as list $isList = in_array($key, ['domain', 'subject', 'resources', 'networks'], true); if ($isList) { $items = is_array($val) ? $val : [$val]; $block .= $prefix . $key . ":\n"; foreach ($items as $item) { $out = $key === 'subject' ? "['" . $item . "']" : vv_authelia_yaml_scalar((string) $item); $block .= ' - ' . $out . "\n"; } } else { $block .= $prefix . $key . ': ' . vv_authelia_yaml_scalar((string) $val) . "\n"; } } } // Replace existing access_control: block (from its line to next top-level key or EOF) $pattern = '/^access_control:[ \t]*\n(?:[ \t][^\n]*\n?)*/m'; $new = preg_match($pattern, $content) ? preg_replace($pattern, $block, $content, 1) : rtrim($content) . "\n\n" . $block; if ($new === null) return ['ok' => false, 'error' => 'Regex replace failed']; $tmp = $file . '.vv.tmp'; if (file_put_contents($tmp, $new) === false) return ['ok' => false, 'error' => 'Write failed']; if (!rename($tmp, $file)) { @unlink($tmp); return ['ok' => false, 'error' => 'Atomic rename failed']; } shell_exec('docker restart ' . escapeshellarg($conf['authelia_container']) . ' >/dev/null 2>&1 &'); return ['ok' => true]; } // Quote a YAML scalar value if it contains characters that require quoting. function vv_authelia_yaml_scalar(string $val): string { if ($val === '' || preg_match('/[:#\[\]{},|>&*?!%@`\'"]/', $val) || preg_match('/^\s|\s$/', $val)) return '"' . str_replace(['\\', '"'], ['\\\\', '\\"'], $val) . '"'; return $val; } // ── Access simulation ───────────────────────────────────────────────────────── // // Whether one person can open one URL is decided by three objects that no single card shows: the // NPM host (does it hand the request to Authelia at all), the Authelia rule list (which rule wins, // in file order), and the LDAP group membership (does the winning rule's subject include them). // Any one of the three can be the reason a login is refused — or worse, not asked for — and the // only way to find out has been to read three configs and reason about them by hand. // // This walks it the way Authelia does and reports every step, so the answer is checkable rather // than asserted. It changes nothing: every function below reads. // Which Authelia a proxy host actually talks to, read out of its own nginx block rather than // assumed from conf. // // This machine runs two — Authelia for the .com names and Authelia-Secondary for the .us ones — // and HOST1_AUTHELIA_CONFIG names only the first. Evaluating a .us domain against the primary's // rules would produce a confident, wrong answer for six live hostnames, so the instance is taken // from the `set $upstream_authelia http://NAME:PORT` line that decides it in production. function vv_authelia_instance_for(array $proxyHost): array { $adv = (string) ($proxyHost['advanced_config'] ?? ''); if (!str_contains($adv, 'auth_request')) return ['guarded' => false, 'container' => '', 'config' => '', 'source' => 'none']; $container = ''; if (preg_match('#set\s+\$upstream_authelia\s+https?://([A-Za-z0-9._-]+):(\d+)#', $adv, $m)) $container = $m[1]; $conf = vv_auth_conf(); // The configured instance is matched by name rather than assumed, so the tab's own editing // target is identified as such and anything else is reported as the separate instance it is. if ($container !== '' && strcasecmp($container, (string) ($conf['authelia_container'] ?? '')) === 0) return ['guarded' => true, 'container' => $container, 'config' => $conf['authelia_config'], 'source' => 'conf', 'is_configured' => true]; $path = $container !== '' ? vv_authelia_config_for_container($container) : ''; return ['guarded' => true, 'container' => $container, 'config' => $path, 'source' => $path !== '' ? 'docker' : 'unknown', 'is_configured' => false]; } // A container's configuration.yml, found through its own /config bind mount. Nothing hardcodes a // path: a second instance added later is picked up because it is mounted the same way, which is the // same reason the rest of this plugin reads its host list from NPM rather than from conf. function vv_authelia_config_for_container(string $name): string { static $cache = []; if (isset($cache[$name])) return $cache[$name]; $cache[$name] = ''; require_once __DIR__ . '/docker.php'; $all = vv_dk_inspect_all(); foreach ($all as $cn => $c) { if (strcasecmp($cn, $name) !== 0) continue; foreach ($c['mounts'] ?? [] as $m) { if (($m['dst'] ?? '') !== '/config') continue; $p = rtrim((string) $m['src'], '/') . '/configuration.yml'; if (is_file($p)) $cache[$name] = $p; } } return $cache[$name]; } // Does an Authelia domain pattern match this hostname? Authelia accepts an exact name and a single // leading wildcard label; nothing else here uses regex domains, so nothing else is claimed. function vv_authelia_domain_matches(string $pattern, string $domain): bool { $pattern = strtolower(trim($pattern)); $domain = strtolower(trim($domain)); if ($pattern === '' || $domain === '') return false; if ($pattern === $domain) return true; if (str_starts_with($pattern, '*.')) { $suffix = substr($pattern, 1); // ".example.com" // One label only, matching Authelia: *.example.com covers a.example.com and not a.b.example.com. return str_ends_with($domain, $suffix) && !str_contains(substr($domain, 0, -strlen($suffix)), '.') && substr($domain, 0, -strlen($suffix)) !== ''; } return false; } // Whether a rule's subject admits this user. Authelia's shape is a list of subjects OR'd together, // where an element that is itself a list is AND'd — so [[a,b],c] means "(a and b) or c". // // A rule with no subject at all applies to everyone, which is the case that silently shadows every // specific rule below it. Returned as a reason string as well as a verdict, because "the rule was // skipped" and "the rule matched and denied" look identical in a result and mean opposite things. function vv_authelia_subject_matches($subject, string $uid, array $groups): array { if ($subject === null || $subject === '' || $subject === []) return ['match' => true, 'why' => 'no subject — applies to everyone']; $groupsLc = array_map('strtolower', $groups); $one = function (string $s) use ($uid, $groupsLc): bool { $s = trim($s); if (str_starts_with($s, 'group:')) return in_array(strtolower(substr($s, 6)), $groupsLc, true); if (str_starts_with($s, 'user:')) return strcasecmp(substr($s, 5), $uid) === 0; // An unprefixed subject is a username in Authelia's schema. return strcasecmp($s, $uid) === 0; }; $alternatives = is_array($subject) ? $subject : [$subject]; foreach ($alternatives as $alt) { if (is_array($alt)) { $all = true; foreach ($alt as $part) if (!$one((string) $part)) { $all = false; break; } if ($all) return ['match' => true, 'why' => 'matches all of ' . implode(' + ', array_map('strval', $alt))]; } elseif ($one((string) $alt)) { return ['match' => true, 'why' => 'matches ' . $alt]; } } $flat = []; foreach ($alternatives as $alt) $flat[] = is_array($alt) ? '(' . implode(' + ', array_map('strval', $alt)) . ')' : (string) $alt; return ['match' => false, 'why' => 'not ' . implode(' or ', $flat)]; } // Walk the rules in file order and stop at the first that matches on every axis, which is exactly // what Authelia does. Every rule considered is reported with why it did or did not apply — the // trace is the point, because "which rule won" is rarely the surprising part. "Which rule you // thought would win and why it was skipped" is. function vv_authelia_evaluate(array $rules, string $defaultPolicy, string $domain, string $path, string $uid, array $groups): array { $trace = []; foreach ($rules as $i => $r) { $doms = is_array($r['domain'] ?? '') ? $r['domain'] : [$r['domain'] ?? '']; $domHit = false; foreach ($doms as $d) if (vv_authelia_domain_matches((string) $d, $domain)) { $domHit = true; break; } $label = trim(preg_replace('/^#+\s*/', '', implode(' ', $r['_label'] ?? []))) ?: ('rule ' . ($i + 1)); if (!$domHit) { $trace[] = ['n' => $i + 1, 'label' => $label, 'applied' => false, 'skip' => 'domain', 'why' => 'domain not listed']; continue; } // Resources is a path regex. A rule carrying one only applies to the paths it names, so a // rule that looks like it covers a host may cover one directory of it. $res = $r['resources'] ?? null; if ($res !== null && $res !== '' && $res !== []) { $list = is_array($res) ? $res : [$res]; $hit = false; foreach ($list as $rx) { // Delimited and error-suppressed: this pattern comes from a hand-edited file, and a // malformed one must report as "did not match" rather than raising a warning into // the answer. if (@preg_match('#' . str_replace('#', '\#', (string) $rx) . '#', $path)) { $hit = true; break; } } if (!$hit) { $trace[] = ['n' => $i + 1, 'label' => $label, 'applied' => false, 'skip' => 'path', 'why' => 'domain matches but the path ' . $path . ' is outside its resources pattern']; continue; } } $sub = vv_authelia_subject_matches($r['subject'] ?? null, $uid, $groups); if (!$sub['match']) { // Recorded as a subject skip specifically. "No rule mentioned this host" and "a rule // for this host stepped over this person" both end at the default policy and are // completely different facts about the configuration. $trace[] = ['n' => $i + 1, 'label' => $label, 'applied' => false, 'skip' => 'subject', 'policy' => $r['policy'] ?? 'deny', 'why' => 'domain matches but the user is ' . $sub['why']]; continue; } $trace[] = ['n' => $i + 1, 'label' => $label, 'applied' => true, 'why' => 'domain matches and the user ' . $sub['why'], 'policy' => $r['policy'] ?? 'deny']; return ['policy' => $r['policy'] ?? 'deny', 'matched' => $i + 1, 'matched_label' => $label, 'trace' => $trace]; } return ['policy' => $defaultPolicy, 'matched' => null, 'matched_label' => '', 'trace' => $trace]; } // The whole question, end to end: can this user open this URL, and what decided it. function vv_auth_access_check(string $domain, string $uid, string $path = '/'): array { $domain = strtolower(trim($domain)); $path = $path === '' ? '/' : $path; if ($domain === '') return ['ok' => false, 'error' => 'No domain given']; $p = vv_npm_list_proxies(); if (!($p['ok'] ?? false)) return ['ok' => false, 'error' => $p['error'] ?? 'Could not read proxy hosts']; $host = null; foreach ($p['proxies'] as $h) foreach ($h['domain_names'] ?? [] as $d) if (vv_authelia_domain_matches((string) $d, $domain) || strtolower((string) $d) === $domain) { $host = $h; break 2; } $out = ['ok' => true, 'domain' => $domain, 'path' => $path, 'uid' => $uid, 'findings' => []]; if (!$host) { $out['findings'][] = ['level' => 'warn', 'text' => 'No NPM proxy host serves ' . $domain . ', so nothing reaches Authelia for it and no rule about it has any effect.']; $out['served'] = false; return $out; } $out['served'] = true; $out['enabled'] = ($host['enabled'] ?? true) ? true : false; $out['forward'] = ($host['forward_scheme'] ?? 'http') . '://' . ($host['forward_host'] ?? '') . ':' . ($host['forward_port'] ?? ''); // The user's groups, which is the half of the answer that lives in a different system entirely. // Taken from the user record rather than by walking every group, because that record carries // both facts this needs — whether the person exists and what they belong to. Asking the group // list instead would make "in no groups" and "no such person" the same empty result. $groups = []; $known = false; if ($uid !== '') { $ul = vv_lldap_list_users(); foreach (($ul['ok'] ?? false) ? ($ul['users'] ?? []) : [] as $u) { if (strcasecmp((string) ($u['id'] ?? ''), $uid) !== 0) continue; $known = true; $groups = array_values(array_filter(array_column($u['groups'] ?? [], 'displayName'))); break; } } $out['groups'] = $groups; $out['user_known'] = $known; $inst = vv_authelia_instance_for($host); $out['authelia'] = $inst; if (!$inst['guarded']) { $out['policy'] = 'bypass'; $out['findings'][] = ['level' => 'warn', 'text' => 'This host has no auth_request block, so the request never reaches Authelia. ' . 'Anyone who can resolve ' . $domain . ' gets through to the application, whatever the rules say.']; return $out; } if ($inst['config'] === '' || !is_file($inst['config'])) { $out['findings'][] = ['level' => 'warn', 'text' => 'This host sends its authentication to ' . ($inst['container'] ?: 'an unnamed instance') . ', whose configuration could not be located, so the decision below cannot be worked out.']; return $out; } $r = vv_authelia_read_rules($inst['config']); if (!($r['ok'] ?? false)) { $out['findings'][] = ['level' => 'warn', 'text' => $r['error'] ?? 'Rules unreadable']; return $out; } $ev = vv_authelia_evaluate($r['rules'] ?? [], $r['default_policy'] ?? 'deny', $domain, $path, $uid, $groups); $out['policy'] = $ev['policy']; $out['matched'] = $ev['matched']; $out['matched_label'] = $ev['matched_label']; $out['trace'] = $ev['trace']; $out['default_policy'] = $r['default_policy'] ?? 'deny'; $out['findings'] = array_merge($out['findings'], vv_auth_access_findings($out, $inst)); return $out; } function vv_auth_access_findings(array $o, array $inst): array { $f = []; // The one that cannot be seen from any single page. A rule list that is not the one being // edited on this tab is a rule list nobody is maintaining on purpose. if (!($inst['is_configured'] ?? false)) $f[] = ['level' => 'info', 'text' => 'Decided by ' . ($inst['container'] ?: 'a second instance') . ', which is not the Authelia this tab edits. Its rules are in ' . ($inst['config'] ?: 'a config that was not found') . ' and nothing on this page changes them.']; if (!($o['enabled'] ?? true)) $f[] = ['level' => 'warn', 'text' => 'The proxy host is disabled in NPM, so nothing is served here at all right now.']; if ($o['uid'] !== '' && !($o['user_known'] ?? false)) $f[] = ['level' => 'warn', 'text' => 'No user with the id "' . $o['uid'] . '" exists in the directory, so this is the answer for a name that cannot log in.']; elseif ($o['uid'] !== '' && !$o['groups']) $f[] = ['level' => 'info', 'text' => $o['uid'] . ' is in no groups, so every rule with a group subject skips them.']; $policy = $o['policy'] ?? ''; if ($o['matched'] === null) { // Two different facts end at the same default policy, and conflating them was the first // version of this: a host no rule mentions, and a host whose rule stepped over this // particular person. The second is the more pointed one — the rule exists, it was written // for this host, and the default let them past it anyway. $stepped = []; foreach ($o['trace'] ?? [] as $t) if (($t['skip'] ?? '') === 'subject') $stepped[] = $t; if ($stepped && $policy === 'bypass') { $t = $stepped[0]; $f[] = ['level' => 'bad', 'text' => 'Rule ' . $t['n'] . ' (' . $t['label'] . ') covers ' . $o['domain'] . ' but does not apply to this user — ' . $t['why'] . '. No later rule matches either, so the default policy takes over, ' . 'and the default here is bypass. The rule written to protect this host lets everyone it does not name straight through.']; // Stated because the answer is different for a caller who is not logged in at all, and // an operator reading "bypass" would otherwise reasonably conclude the host is open to // the internet. Authelia treats an anonymous request against a rule carrying a subject // as a potential match and sends them to the portal first; this simulation answers for // someone who has already authenticated as this user. $f[] = ['level' => 'info', 'text' => 'This is the answer for a caller already logged in as ' . ($o['uid'] ?: 'someone') . '. Authelia handles an anonymous caller differently — a rule carrying a subject makes it send them to the login portal first — ' . 'so this is an authenticated user reaching something not meant for them, not an open door to the internet.']; } elseif ($policy === 'bypass') { $f[] = ['level' => 'bad', 'text' => 'No rule mentions ' . $o['domain'] . ' at all, so it falls to the default policy, which is bypass — ' . 'the request goes to Authelia and Authelia waves it through. This host is behind an auth_request block that never refuses anyone.']; } else { $f[] = ['level' => 'info', 'text' => 'No rule matches ' . $o['domain'] . ', so the default policy of ' . $policy . ' applies.']; } } else { $f[] = ['level' => $policy === 'deny' ? 'warn' : 'info', 'text' => 'Rule ' . $o['matched'] . ' (' . $o['matched_label'] . ') is the first one that applies, and its policy is ' . $policy . '.']; } if ($policy === 'bypass' || $policy === '') $f[] = ['level' => 'info', 'text' => ($o['uid'] !== '' ? $o['uid'] : 'Anyone') . ' reaches ' . $o['domain'] . ' without being asked to authenticate. Whether that is right depends on whether the application behind it has its own login.']; elseif ($policy === 'deny') $f[] = ['level' => 'warn', 'text' => ($o['uid'] !== '' ? $o['uid'] : 'This caller') . ' is refused before reaching the application.']; else $f[] = ['level' => 'info', 'text' => ($o['uid'] !== '' ? $o['uid'] : 'A caller') . ' is asked to log in (' . $policy . ') and then reaches the application.']; return $f; } // ── Certificate renewal triage ──────────────────────────────────────────────── // // Why renewals failed, from certbot's own logs. cert_history.sh counts failures by noticing an // expiry in the past; this reads the reason. See Tools/cert_triage.php for the full note. // The categories renewal failures actually fall into here, in the order a reader should meet them: // causes before consequences. Each pattern is anchored on the string certbot itself emits, so a // category matching is evidence rather than inference. // // 'root' marks a cause worth acting on directly. Rate limiting is deliberately not one — it is // what happens after something else has been failing, and treating it as the problem sends people // to wait out a timer instead of fixing the DNS record that burned it. const VV_CERT_TRIAGE_PATTERNS = [ ['id' => 'no-dns-record', 'root' => true, 'rx' => '/DNS problem: NXDOMAIN looking up [A-Z]+ for ([A-Za-z0-9._-]+)/', 'what' => 'the hostname has no DNS record at all'], ['id' => 'no-a-record', 'root' => true, 'rx' => '/no valid A records found for ([A-Za-z0-9._-]+)/', 'what' => 'the hostname resolves but has no address record Let\'s Encrypt can reach'], ['id' => 'challenge-unreachable', 'root' => true, 'rx' => '/Timeout during connect[^\n]*|Fetching http:\/\/([A-Za-z0-9._-]+)\/\.well-known[^\n]*Timeout/', 'what' => 'the HTTP-01 challenge could not be fetched — port 80 is not reaching this proxy'], ['id' => 'caa-forbids', 'root' => true, 'rx' => '/CAA record for ([A-Za-z0-9._-]+) prevents issuance/', 'what' => 'a CAA record on the domain forbids Let\'s Encrypt from issuing'], ['id' => 'revoke-expired', 'root' => false, 'rx' => '/Unable to revoke :: Certificate is expired/', 'what' => 'a revoke was attempted on a certificate that had already expired'], // Last, and marked as a consequence. 2079 of these on this installation, every one of them // downstream of the three hostnames above. ['id' => 'rate-limited', 'root' => false, 'rx' => '/urn:ietf:params:acme:error:rateLimited/', 'what' => 'Let\'s Encrypt refused the request because too many were made too recently'], ]; // The rotated certbot logs, newest first, ordered by their rotation suffix. // // Never by mtime. Every one of these files carries the same mtime on this machine — they live in // the Critical-Data share and are written as a set by the sync, so the filesystem says all // thousand were modified in the same minute. Sorting by mtime picks an arbitrary sample from // anywhere in the history and presents it as the current state. function vv_cert_log_files(int $limit): array { $dir = trim((string) (vv_conf_vars()['CERT_TRIAGE_LOG_DIR'] ?? '')); if ($dir === '') { require_once __DIR__ . '/docker.php'; foreach (vv_dk_inspect_all() as $name => $c) { if (stripos($name, 'nginx') === false && stripos($name, 'npm') === false) continue; foreach ($c['mounts'] ?? [] as $m) { if (($m['dst'] ?? '') !== '/config' && ($m['dst'] ?? '') !== '/data') continue; $p = rtrim((string) $m['src'], '/') . '/log'; if (is_dir($p)) { $dir = $p; break 2; } } } } if ($dir === '' || !is_dir($dir)) return ['dir' => $dir, 'files' => []]; $found = glob($dir . '/letsencrypt.log*') ?: []; $rank = []; foreach ($found as $f) { // letsencrypt.log is the live one and sorts ahead of every numbered rotation. $n = preg_match('/\.log\.(\d+)$/', $f, $m) ? (int) $m[1] : -1; $rank[$f] = $n; } asort($rank); return ['dir' => $dir, 'files' => array_slice(array_keys($rank), 0, max(1, $limit))]; } // The tail of a file, without reading the whole thing. These run to a megabyte each and the // interesting part of a certbot run is always at the end. function vv_cert_log_tail(string $path, int $maxBytes): string { $size = @filesize($path); if ($size === false) return ''; $fh = @fopen($path, 'rb'); if (!$fh) return ''; if ($size > $maxBytes) @fseek($fh, $size - $maxBytes); $data = (string) @stream_get_contents($fh); fclose($fh); return $data; } function vv_cert_triage(int $filesOverride = 0): array { $v = vv_conf_vars(); $lim = $filesOverride > 0 ? $filesOverride : max(1, (int) ($v['CERT_TRIAGE_FILES'] ?? 40)); $max = max(4096, (int) ($v['CERT_TRIAGE_MAX_BYTES'] ?? 262144)); $found = vv_cert_log_files($lim); if (!$found['files']) return ['ok' => false, 'error' => 'No certbot logs found' . ($found['dir'] !== '' ? ' in ' . $found['dir'] : ' — set CERT_TRIAGE_LOG_DIR')]; // Counted per run, not per line. One log file is one certbot invocation, and a single failed // run writes its reason several times over — in the ACME response, in the traceback, and again // in certbot's own ERROR summary. Counting lines therefore reports one failure as three and // makes the categories incomparable with each other, because the noisier reasons repeat more. // "12 of 40 runs failed for no DNS record" is a number that means something. $counts = $doms = []; $runs = $failedRuns = $unclassified = 0; foreach ($found['files'] as $f) { $text = vv_cert_log_tail($f, $max); if ($text === '') continue; $runs++; $hitAny = false; foreach (VV_CERT_TRIAGE_PATTERNS as $p) { if (!preg_match_all($p['rx'], $text, $m, PREG_SET_ORDER)) continue; $hitAny = true; $counts[$p['id']] = ($counts[$p['id']] ?? 0) + 1; foreach ($m as $hit) // Only some patterns capture a hostname; the others are about the run, not a name. if (isset($hit[1]) && $hit[1] !== '') $doms[$p['id']][strtolower($hit[1])] = true; } if ($hitAny) $failedRuns++; // A run that errored and matched nothing known. Reported rather than dropped: an error // certbot starts emitting after this was written has to show up as something, and a total // that quietly shrinks is how a new failure mode stays invisible. if (!$hitAny && preg_match('/:ERROR:certbot/', $text)) { $unclassified++; $failedRuns++; } } $cats = []; foreach (VV_CERT_TRIAGE_PATTERNS as $p) { if (empty($counts[$p['id']])) continue; $cats[] = ['id' => $p['id'], 'root' => $p['root'], 'what' => $p['what'], 'count' => $counts[$p['id']], 'domains' => array_keys($doms[$p['id']] ?? [])]; } return ['ok' => true, 'dir' => $found['dir'], 'files_read' => $runs, 'total' => $failedRuns, 'unclassified' => $unclassified, 'categories' => $cats, 'reading' => vv_cert_triage_reading($cats)]; } // The causal reading, which is the part a category count cannot give. Written as sentences because // the relationship between these categories is the whole finding: one of them is nearly always // downstream of another, and a list sorted by count puts the consequence at the top. function vv_cert_triage_reading(array $cats): array { if (!$cats) return []; $by = []; foreach ($cats as $c) $by[$c['id']] = $c; $roots = array_values(array_filter($cats, fn($c) => $c['root'])); $out = []; if ($roots) { $names = []; foreach ($roots as $r) foreach ($r['domains'] as $d) $names[$d] = true; $n = count($roots); $out[] = 'Root cause: ' . $n . ' kind' . ($n === 1 ? '' : 's') . ' of failure that ' . ($n === 1 ? 'is' : 'are') . ' nobody else\'s consequence' . ($names ? ', affecting ' . implode(', ', array_slice(array_keys($names), 0, 8)) . (count($names) > 8 ? ' and ' . (count($names) - 8) . ' more' : '') : '') . '.'; } if (isset($by['rate-limited'])) { $out[] = $roots ? 'The ' . $by['rate-limited']['count'] . ' rate-limit refusals are downstream of that: ' . 'certbot retried the failing names often enough to exhaust the allowance, which then ' . 'fails renewals for domains that have nothing wrong with them. Fixing the names above ' . 'is what clears it — waiting out the limit only restarts the cycle.' : 'Rate limiting is the only category present, with no failing name behind it. That points ' . 'at renewal being attempted far too often rather than at any one domain.'; } if (isset($by['revoke-expired'])) $out[] = 'The revoke failures are harmless in themselves — a certificate that already expired ' . 'cannot be revoked, and does not need to be. They indicate cleanup running against ' . 'certificates that were already dead.'; if (!$roots && !isset($by['rate-limited'])) $out[] = 'Nothing here is a standing cause — these are individual failures rather than a pattern.'; return $out; }