Compare commits
2
Commits
1001c25487
...
01601d210b
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
01601d210b | ||
|
|
99b58c0c4f |
@@ -494,6 +494,11 @@
|
||||
# per-domain renewal and failure counts.
|
||||
"Plugin/unraid/Tools/cert_history.sh" # record cert renewals, failures and age per domain
|
||||
"Plugin/unraid/Tools/npm_access_stats.sh" # aggregate NPM per-host access logs into request and byte totals
|
||||
# After both of the above, because it reads what they write — the uptime history and the
|
||||
# per-host request totals are two of the four things it reasons from. Daily rather than
|
||||
# hourly: everything it files is a condition that has already lasted hours by the time
|
||||
# AUTH_SWEEP_DOWN_MIN lets it through, so a faster cadence would find nothing new.
|
||||
"Plugin/unraid/Tools/auth_sweep.sh" # file findings for hosts that are not serving, and hostnames Authelia is not protecting
|
||||
)
|
||||
|
||||
# Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart.
|
||||
@@ -1498,6 +1503,40 @@
|
||||
UPTIME_PROBE_TIMEOUT=8 # seconds per domain before it counts as down
|
||||
UPTIME_PROBE_LIST_TTL=900 # seconds to reuse the domain list from NPM before re-reading it
|
||||
|
||||
# ── Auth Sweep ──
|
||||
# Tools/auth_sweep.sh asks the two questions the Auth tab answers about one host, about every host,
|
||||
# and files what it finds as findings on the AI tab. Reports only — it starts nothing and rewrites
|
||||
# nothing, because every remedy here (start a container, edit a rule, change a default policy) is a
|
||||
# decision rather than a correction.
|
||||
#
|
||||
# proxy_down a host below UPTIME_MIN that has been failing longer than DOWN_MIN. The time gate
|
||||
# is what keeps a reboot from filing a finding for every hostname on the machine.
|
||||
# access_open an Authelia instance whose default policy lets through every hostname its rules do
|
||||
# not decide. Filed once per instance, not once per hostname — they all have the
|
||||
# same single fix, and one finding per name is twenty-two copies of one sentence.
|
||||
#
|
||||
# The access half reads whichever Authelia each proxy host actually points at, which is not always
|
||||
# the one HOST*_AUTHELIA_CONFIG names — this installation runs two.
|
||||
AUTH_SWEEP_ENABLED=true # master switch
|
||||
AUTH_SWEEP_UPTIME_MIN=96 # 24h percentage below which a host becomes a candidate
|
||||
AUTH_SWEEP_DOWN_MIN=120 # minutes it must have been failing before anything is filed
|
||||
AUTH_SWEEP_ACCESS_CHECK=true # run the "is it actually protected" half at all
|
||||
|
||||
# ── Cert Triage ──
|
||||
# Tools/cert_triage.sh reads certbot's own logs and names why renewals failed. cert_history.sh
|
||||
# counts failures by noticing an expiry in the past; this reads the reason.
|
||||
#
|
||||
# Counts runs, not lines: one log file is one certbot invocation, and one failure writes its
|
||||
# reason into the ACME response, the traceback and certbot's summary, so line counting reports it
|
||||
# three times and inflates whichever category is most verbose.
|
||||
#
|
||||
# The log directory is found from the NPM container's own mount. Set CERT_TRIAGE_LOG_DIR only if
|
||||
# that lookup cannot work. Both bounds exist because this is reachable from a page request and the
|
||||
# directory here is 639 MB across a thousand rotated files.
|
||||
CERT_TRIAGE_FILES=40 # rotated logs to read, newest first by rotation suffix
|
||||
CERT_TRIAGE_MAX_BYTES=262144 # bytes read from the end of each — a run's reason is always last
|
||||
CERT_TRIAGE_LOG_DIR="" # empty = find it from the NPM container
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Verifies rsync mirror health by comparing random file checksums between servers.
|
||||
# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect.
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Looks at the whole auth stack the way the Auth tab's two checks look at one thing, and files
|
||||
// what it finds. Two questions, asked of every host NPM serves:
|
||||
//
|
||||
// Is this host actually serving? — the why-check, for anything below threshold
|
||||
// Is this host actually protected? — the access check, for anything behind Authelia
|
||||
//
|
||||
// WHY IT EXISTS
|
||||
// Both answers already existed on demand, and both required somebody to open the Auth tab and
|
||||
// press a button on the right row. Nobody does that on a working day. One host here has returned
|
||||
// nothing but 5xx to every request for months, and the hostnames that sit behind an auth_request
|
||||
// block which then waves everyone through are invisible from every page in the plugin, because
|
||||
// the fact is split across NPM, an Authelia config and the directory.
|
||||
//
|
||||
// Findings are the existing answer to "a condition that persists and nobody is looking" — the AI
|
||||
// tab already lists, grades, ages out and notifies on them. This adds the auth stack as a source
|
||||
// rather than inventing a second place for the same idea.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// One pass = read NPM once, then per host: the recorded uptime, and — only where it is warranted
|
||||
// — the live checks. The expensive part is deliberately gated:
|
||||
//
|
||||
// proxy_down filed when 24h uptime is below AUTH_SWEEP_UPTIME_MIN *and* the host has been in
|
||||
// that state longer than AUTH_SWEEP_DOWN_MIN. A restart or a reboot produces a
|
||||
// perfectly ordinary dip, and a finding for every one of those is a list nobody
|
||||
// reads. Live checks run only for hosts that pass this gate.
|
||||
//
|
||||
// access_open filed when a host carries an auth_request block and the Authelia deciding it
|
||||
// reaches its default policy of bypass for a member of no relevant group. No
|
||||
// network calls at all — this is three config files compared.
|
||||
//
|
||||
// Nothing is repaired, restarted or rewritten. This is a reporting pass, and it stays one: the
|
||||
// remedies here are "start a container", "edit a rule", "change a default policy", and every one
|
||||
// of them is a decision rather than a correction.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// The gate is time, not count.
|
||||
// A host that has been down for four hours is news whether the probe caught eight samples
|
||||
// or eight hundred. Basing it on samples would file findings at a different threshold on a
|
||||
// node whose probe runs at a different cadence.
|
||||
//
|
||||
// Findings are refreshed, not duplicated.
|
||||
// vv_ai_finding_write() keys on kind + subject + ref, so a host still down tomorrow updates
|
||||
// the record it filed today rather than adding a second one. The store closes what stops
|
||||
// recurring on its own.
|
||||
//
|
||||
// The access check answers for a real user, not a hypothetical one.
|
||||
// "Does the default policy let somebody in" is only meaningful about a person who exists.
|
||||
// The sweep asks about a directory member holding no privileged group, because that is the
|
||||
// account that reveals a rule which protects nothing.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Non-fatal, always. No NPM, no credentials, no domains, AI repair switched off — exits 0.
|
||||
// One pass at a time, flock non-blocking, so a slow pass cannot overlap the next.
|
||||
// Live probing is bounded by the gate above, so a total outage cannot turn one pass into
|
||||
// thirty-five sequential timeouts.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// auth_sweep.php one pass, files findings
|
||||
// auth_sweep.php --dry-run report what it would file, write nothing
|
||||
// auth_sweep.php --report one-screen summary for the Sunday report; silent when clean
|
||||
//
|
||||
// CONFIGURATION
|
||||
// AUTH_SWEEP_ENABLED master switch (default true)
|
||||
// AUTH_SWEEP_UPTIME_MIN 24h percentage below which a host is a candidate (default 96)
|
||||
// AUTH_SWEEP_DOWN_MIN minutes it must have been failing before filing (default 120)
|
||||
// AUTH_SWEEP_ACCESS_CHECK whether to run the protection half at all (default true)
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
require_once dirname(__DIR__) . '/include/auth.php';
|
||||
require_once dirname(__DIR__) . '/include/ai_repair.php';
|
||||
|
||||
$dryRun = in_array('--dry-run', $argv, true);
|
||||
$report = in_array('--report', $argv, true);
|
||||
|
||||
$lock = @fopen(sys_get_temp_dir() . '/vv_auth_sweep.lock', 'c');
|
||||
if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) { echo "another pass is running\n"; exit(0); }
|
||||
|
||||
try {
|
||||
$v = vv_conf_vars();
|
||||
if (strtolower(trim($v['AUTH_SWEEP_ENABLED'] ?? 'true')) === 'false') {
|
||||
echo "AUTH_SWEEP_ENABLED is false\n"; exit(0);
|
||||
}
|
||||
$minPct = (float) ($v['AUTH_SWEEP_UPTIME_MIN'] ?? 96);
|
||||
$minDown = max(1, (int) ($v['AUTH_SWEEP_DOWN_MIN'] ?? 120)) * 60;
|
||||
$doAccess = strtolower(trim($v['AUTH_SWEEP_ACCESS_CHECK'] ?? 'true')) !== 'false';
|
||||
|
||||
$p = vv_npm_list_proxies();
|
||||
if (!($p['ok'] ?? false)) { echo 'NPM: ' . ($p['error'] ?? 'unreadable') . "\n"; exit(0); }
|
||||
|
||||
$uptime = is_file(vv_auth_db_file('uptime.json'))
|
||||
? (json_decode((string) @file_get_contents(vv_auth_db_file('uptime.json')), true) ?: []) : [];
|
||||
$doms = $uptime['domains'] ?? [];
|
||||
|
||||
$filed = $skipped = 0;
|
||||
$lines = [];
|
||||
|
||||
// ── Half one: hosts that are not serving ──
|
||||
foreach ($p['proxies'] as $h) {
|
||||
// A host switched off is not a fault, it is a decision. Filing one would put a finding on
|
||||
// the list for every host the operator has deliberately parked.
|
||||
if (($h['enabled'] ?? true) === false) continue;
|
||||
|
||||
$names = [];
|
||||
foreach ($h['domain_names'] ?? [] as $d) {
|
||||
$d = strtolower(trim((string) $d));
|
||||
if ($d !== '' && !str_contains($d, '*')) $names[] = $d;
|
||||
}
|
||||
if (!$names) continue;
|
||||
|
||||
// The worst name decides, for the same reason the row does: a host is only as reachable as
|
||||
// its least reachable hostname, and averaging hides one dead name behind three healthy ones.
|
||||
$worstPct = null; $worstDom = ''; $worstRec = null;
|
||||
foreach ($names as $d) {
|
||||
$r = $doms[$d] ?? null;
|
||||
if (!$r) continue;
|
||||
$pct = vv_auth_uptime_window($r['hours'] ?? [], 24);
|
||||
if ($pct === null) continue;
|
||||
if ($worstPct === null || $pct < $worstPct) { $worstPct = $pct; $worstDom = $d; $worstRec = $r; }
|
||||
}
|
||||
if ($worstPct === null || $worstPct >= $minPct) continue;
|
||||
|
||||
// How long it has actually been like this. last_change is when the state last flipped, so
|
||||
// a host that went down two minutes ago is excluded here and caught on a later pass — which
|
||||
// is the whole point of the gate.
|
||||
$since = (int) ($worstRec['last_change'] ?? 0);
|
||||
$for = $since > 0 ? time() - $since : 0;
|
||||
if (($worstRec['state'] ?? '') === 'down' && $for < $minDown) { $skipped++; continue; }
|
||||
|
||||
$why = vv_npm_why((int) ($h['id'] ?? 0));
|
||||
if (!($why['ok'] ?? false)) continue;
|
||||
|
||||
// The findings the check already writes, which is the whole reason this does not have its
|
||||
// own opinion about what is wrong. Only the decisive ones are carried into the record.
|
||||
$said = [];
|
||||
foreach ($why['findings'] as $f) if (in_array($f['level'], ['bad', 'warn'], true)) $said[] = $f['text'];
|
||||
if (!$said) { $skipped++; continue; }
|
||||
|
||||
$evidence = sprintf("%s is at %.2f%% over 24h%s.\n\n%s",
|
||||
$worstDom, $worstPct,
|
||||
$for > 0 ? ' and has been ' . ($worstRec['state'] ?? 'failing') . ' for ' . round($for / 3600, 1) . ' hours' : '',
|
||||
implode("\n", $said));
|
||||
|
||||
$lines[] = sprintf(' %-34s %6.2f%% %s', $worstDom, $worstPct, $said[0]);
|
||||
if ($dryRun) { $filed++; continue; }
|
||||
|
||||
$w = vv_ai_finding_write([
|
||||
'kind' => 'proxy_down',
|
||||
'subject' => implode(', ', $names),
|
||||
'ref' => 'npm:proxy:' . ($h['id'] ?? 0),
|
||||
'evidence' => $evidence,
|
||||
'observed' => sprintf('%.2f%% over 24h', $worstPct),
|
||||
// Proven, because these are measurements rather than an inference: a TCP connect either
|
||||
// completed or it did not, and the access log either counted 5xx or it did not.
|
||||
'proven' => true,
|
||||
]);
|
||||
$w['ok'] ? $filed++ : $skipped++;
|
||||
}
|
||||
|
||||
// ── Half two: hosts that are guarded but not protected ──
|
||||
//
|
||||
// Grouped by Authelia instance, not filed per hostname. The first version of this produced
|
||||
// twenty-two findings that were all the same sentence, because they all had the same cause: a
|
||||
// default policy of bypass means every host whose rule does not name your group lets you
|
||||
// through, so the number of findings was really the number of hostnames. One finding per
|
||||
// instance, naming the hosts it affects, is the fact — and it has one fix rather than
|
||||
// twenty-two.
|
||||
$openLines = [];
|
||||
if ($doAccess) {
|
||||
// Somebody who exists and holds none of the groups the rules name. A rule that still lets
|
||||
// this account through is a rule protecting nothing, and asking about an invented username
|
||||
// would prove nothing about the directory.
|
||||
$probe = vv_auth_sweep_ordinary_user();
|
||||
$byInstance = [];
|
||||
|
||||
foreach ($p['proxies'] as $h) {
|
||||
if (($h['enabled'] ?? true) === false) continue;
|
||||
if (!str_contains((string) ($h['advanced_config'] ?? ''), 'auth_request')) continue;
|
||||
|
||||
foreach ($h['domain_names'] ?? [] as $d) {
|
||||
$d = strtolower(trim((string) $d));
|
||||
if ($d === '' || str_contains($d, '*')) continue;
|
||||
|
||||
$a = vv_auth_access_check($d, $probe);
|
||||
if (!($a['ok'] ?? false)) continue;
|
||||
if (($a['policy'] ?? '') !== 'bypass') continue;
|
||||
|
||||
$inst = (string) ($a['authelia']['container'] ?? '?');
|
||||
$byInstance[$inst]['default'] = (string) ($a['default_policy'] ?? '?');
|
||||
$byInstance[$inst]['config'] = (string) ($a['authelia']['config'] ?? '');
|
||||
// Which of the two shapes this is, per host: a hostname no rule mentions, or one a
|
||||
// rule covers and then steps over. They have different fixes — write a rule, or
|
||||
// widen an existing one — so the record keeps them apart.
|
||||
$stepped = false;
|
||||
foreach ($a['trace'] ?? [] as $t) if (($t['skip'] ?? '') === 'subject') $stepped = true;
|
||||
$byInstance[$inst][$stepped ? 'stepped' : 'unlisted'][] = $d;
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($byInstance as $inst => $g) {
|
||||
$unlisted = $g['unlisted'] ?? [];
|
||||
$stepped = $g['stepped'] ?? [];
|
||||
$all = array_merge($unlisted, $stepped);
|
||||
if (!$all) continue;
|
||||
|
||||
$ev = [];
|
||||
$ev[] = $inst . ' has default_policy: ' . ($g['default'] ?? '?') . ', so any request its rules do '
|
||||
. 'not decide is allowed through. ' . count($all) . ' hostname'
|
||||
. (count($all) === 1 ? '' : 's') . ' behind an auth_request block pointing at it reach the '
|
||||
. 'application without being asked to authenticate.';
|
||||
if ($unlisted) $ev[] = "\nNo rule mentions these at all:\n " . implode("\n ", $unlisted);
|
||||
if ($stepped) $ev[] = "\nA rule covers these but does not apply to an ordinary account"
|
||||
. ($probe ? ' (tested as ' . $probe . ')' : '') . ":\n " . implode("\n ", $stepped);
|
||||
$ev[] = "\nRules are in " . ($g['config'] ?: 'a config that was not found') . '.';
|
||||
// Said plainly because "bypass" reads as harmless and it is the single most
|
||||
// consequential line in that file.
|
||||
$ev[] = "\nThe fix is a default_policy of deny with an explicit rule for anything that is "
|
||||
. "meant to be public — not a rule per hostname above.";
|
||||
|
||||
$openLines[] = sprintf(' %-22s default bypass — %d hostname%s unprotected',
|
||||
$inst, count($all), count($all) === 1 ? '' : 's');
|
||||
if ($dryRun) { $filed++; continue; }
|
||||
|
||||
$w = vv_ai_finding_write([
|
||||
'kind' => 'access_open',
|
||||
'subject' => $inst . ' — ' . count($all) . ' hostnames not protected',
|
||||
'ref' => 'authelia:' . $inst . ':default_policy',
|
||||
'evidence' => implode("\n", $ev),
|
||||
'observed' => 'default_policy: ' . ($g['default'] ?? '?'),
|
||||
'proven' => true,
|
||||
]);
|
||||
$w['ok'] ? $filed++ : $skipped++;
|
||||
}
|
||||
}
|
||||
|
||||
if ($report) {
|
||||
// Silent on a clean week. The orchestrator's job is to say nothing when there is nothing
|
||||
// to say, and a section that always prints is a section that stops being read.
|
||||
if (!$lines && !$openLines) exit(0);
|
||||
echo "Auth stack review\n";
|
||||
if ($lines) { echo "\nProxy hosts not serving:\n"; foreach ($lines as $l) echo "$l\n"; }
|
||||
if ($openLines) { echo "\nBehind Authelia but not protected:\n"; foreach ($openLines as $l) echo "$l\n"; }
|
||||
exit(1);
|
||||
}
|
||||
|
||||
printf("%s%d finding%s, %d skipped\n", $dryRun ? 'dry run — ' : '', $filed, $filed === 1 ? '' : 's', $skipped);
|
||||
foreach (array_merge($lines, $openLines) as $l) echo "$l\n";
|
||||
exit(0);
|
||||
|
||||
} finally {
|
||||
flock($lock, LOCK_UN);
|
||||
fclose($lock);
|
||||
}
|
||||
|
||||
// A directory member holding none of the groups any rule names. Returns '' when every user is
|
||||
// privileged or the directory cannot be read, and the caller then asks about no user at all —
|
||||
// which still answers the "no rule mentions this host" case and simply cannot answer the
|
||||
// "the rule stepped over this person" one.
|
||||
function vv_auth_sweep_ordinary_user(): string {
|
||||
$named = [];
|
||||
// Groups named by the rules of every Authelia instance in play, not just the configured one —
|
||||
// a .us hostname is decided by a config this conf file does not point at.
|
||||
foreach (vv_auth_sweep_configs() as $cfg) {
|
||||
$r = vv_authelia_read_rules($cfg);
|
||||
foreach (($r['ok'] ?? false) ? $r['rules'] : [] as $rule) {
|
||||
$s = $rule['subject'] ?? null;
|
||||
foreach (is_array($s) ? $s : [$s] as $alt)
|
||||
foreach (is_array($alt) ? $alt : [$alt] as $one)
|
||||
if (is_string($one) && str_starts_with($one, 'group:')) $named[strtolower(substr($one, 6))] = true;
|
||||
}
|
||||
}
|
||||
|
||||
$u = vv_lldap_list_users();
|
||||
foreach (($u['ok'] ?? false) ? $u['users'] : [] as $user) {
|
||||
$mine = array_map('strtolower', array_filter(array_column($user['groups'] ?? [], 'displayName')));
|
||||
if (array_intersect($mine, array_keys($named))) continue;
|
||||
return (string) ($user['id'] ?? '');
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
// Every Authelia configuration this installation actually uses, discovered through the proxy hosts
|
||||
// rather than listed anywhere. Two instances run here and conf names one.
|
||||
function vv_auth_sweep_configs(): array {
|
||||
$out = [];
|
||||
$p = vv_npm_list_proxies();
|
||||
foreach (($p['ok'] ?? false) ? $p['proxies'] : [] as $h) {
|
||||
$i = vv_authelia_instance_for($h);
|
||||
if (($i['config'] ?? '') !== '' && is_file($i['config'])) $out[$i['config']] = true;
|
||||
}
|
||||
return array_keys($out);
|
||||
}
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ===================================== Auth Sweep =============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Asks the two questions the Auth tab can answer about one host, about every host, and files
|
||||
# what it finds as findings.
|
||||
#
|
||||
# Is this host serving? below the uptime threshold, and failing for longer than a restart
|
||||
# Is this host protected? behind an auth_request block that no rule then applies to
|
||||
#
|
||||
# Both answers existed already and both needed somebody to open the tab and press a button on
|
||||
# the right row. One host here has returned nothing but 5xx for months.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# A wrapper. The work is in auth_sweep.php.
|
||||
#
|
||||
# Reports only — nothing is started, restarted or rewritten. The remedies are "start a
|
||||
# container", "edit a rule", "change a default policy", and each of those is a decision.
|
||||
#
|
||||
# The live half is gated on time rather than on sample count: a host must have been failing for
|
||||
# longer than AUTH_SWEEP_DOWN_MIN before anything is filed, so a reboot does not produce a
|
||||
# finding for every hostname on the machine.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# auth_sweep.sh one pass, files findings
|
||||
# auth_sweep.sh --dry-run report what it would file, write nothing
|
||||
# auth_sweep.sh --report one-screen summary for the Sunday report; silent when clean
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# AUTH_SWEEP_ENABLED master switch
|
||||
# AUTH_SWEEP_UPTIME_MIN 24h percentage below which a host is a candidate
|
||||
# AUTH_SWEEP_DOWN_MIN minutes it must have been failing before a finding is filed
|
||||
# AUTH_SWEEP_ACCESS_CHECK whether to run the protection half at all
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
php "$SCRIPT_DIR/auth_sweep.php" "$@"
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Reads certbot's own logs and names why renewals failed, in the handful of categories they
|
||||
// actually fall into — rather than leaving 639 MB of Python tracebacks as the only record.
|
||||
//
|
||||
// WHY IT EXISTS
|
||||
// Tools/cert_history.sh counts failures. It infers them from an expiry in the past, so it knows
|
||||
// that a domain stopped renewing and nothing about why. The why is in certbot's log, which on
|
||||
// this installation is 1001 rotated files, and the answer to "why did ten certificates stop
|
||||
// renewing" was previously a person reading them by hand.
|
||||
//
|
||||
// The categories matter more than the count, because they are not independent. Missing DNS
|
||||
// produces a failure; the failure is retried; the retries exhaust Let's Encrypt's rate limit;
|
||||
// and the rate limit then fails every *other* domain too. A count says "2079 rate limit errors"
|
||||
// and points at the symptom. The chain says "three hostnames have no DNS records, and that is
|
||||
// what burned the rate limit for everything else".
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Reads the newest N log files and classifies each one. One file is one certbot run, and a run
|
||||
// is what gets counted — a single failure writes its reason into the ACME response, the Python
|
||||
// traceback and certbot's own summary, so counting lines reports one failure as three and makes
|
||||
// the noisier categories look larger than the quiet ones. Bounded three ways, because this can
|
||||
// be called from a page request:
|
||||
//
|
||||
// files CERT_TRIAGE_FILES, newest first
|
||||
// bytes CERT_TRIAGE_MAX_BYTES per file, read from the end
|
||||
// order by the numeric rotation suffix, never by mtime
|
||||
//
|
||||
// The suffix is load-bearing. Every one of these files carries the same mtime here — they are
|
||||
// synced as a set, so the filesystem timestamps say they were all written at once. Sorting by
|
||||
// mtime would pick an arbitrary thousand-file-old sample and report it as current.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Classify, never guess. A run that errored and matched no known pattern is counted as
|
||||
// unclassified and said so, rather than being folded into the nearest category — an "unknown"
|
||||
// that is honest is worth more than a tidy chart that is wrong.
|
||||
//
|
||||
// The root causes and the consequences are reported separately. Rate limiting is almost always
|
||||
// downstream of something else here, and listing it alongside its own cause invites fixing the
|
||||
// symptom.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// cert_triage.php summary — categories, affected domains, and the causal reading
|
||||
// cert_triage.php --json the same as JSON, for the Certs tab
|
||||
// cert_triage.php --files=N override how many rotated logs to read
|
||||
//
|
||||
// CONFIGURATION
|
||||
// CERT_TRIAGE_FILES rotated logs to read, newest first (default 40)
|
||||
// CERT_TRIAGE_MAX_BYTES bytes read from the end of each (default 262144)
|
||||
// CERT_TRIAGE_LOG_DIR override the log directory; normally found from the NPM container
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
require_once dirname(__DIR__) . '/include/auth.php';
|
||||
|
||||
$json = in_array('--json', $argv, true);
|
||||
$filesOverride = 0;
|
||||
foreach ($argv as $a) if (preg_match('/^--files=(\d+)$/', $a, $m)) $filesOverride = (int) $m[1];
|
||||
|
||||
$r = vv_cert_triage($filesOverride);
|
||||
|
||||
if ($json) { echo json_encode($r), "\n"; exit(0); }
|
||||
|
||||
if (!($r['ok'] ?? false)) { echo ($r['error'] ?? 'failed'), "\n"; exit(0); }
|
||||
|
||||
printf("%d certbot runs read, %d failed, %d of those matched nothing known\n\n",
|
||||
$r['files_read'], $r['total'], $r['unclassified']);
|
||||
|
||||
if (!$r['total']) { echo "No renewal failures in the logs read.\n"; exit(0); }
|
||||
|
||||
foreach ($r['categories'] as $c) {
|
||||
printf(" %-22s %5d %s\n", $c['id'], $c['count'], $c['what']);
|
||||
foreach (array_slice($c['domains'], 0, 6) as $d) printf(" %s\n", $d);
|
||||
if (count($c['domains']) > 6) printf(" … and %d more\n", count($c['domains']) - 6);
|
||||
}
|
||||
|
||||
if ($r['reading']) { echo "\n"; foreach ($r['reading'] as $l) echo " $l\n"; }
|
||||
exit(0);
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ==================================== Cert Triage =============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Reads certbot's own logs and names why renewals failed, in the handful of categories they
|
||||
# actually fall into.
|
||||
#
|
||||
# cert_history.sh counts failures — it notices an expiry in the past. It cannot say why. The why
|
||||
# is in certbot's log, which here is 1001 rotated files and 639 MB, and the last time anyone
|
||||
# answered "why did ten certificates stop renewing" they read them by hand.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# A wrapper. The work is in cert_triage.php.
|
||||
#
|
||||
# Counts runs, not lines. One log file is one certbot invocation, and one failure writes its
|
||||
# reason three times — in the ACME response, the traceback, and certbot's own summary. Counting
|
||||
# lines reports a single failure as three and inflates whichever category is most verbose.
|
||||
#
|
||||
# Reads the newest logs by rotation suffix, never by mtime. Every file here carries the same
|
||||
# mtime because they are synced as a set, so mtime order is meaningless.
|
||||
#
|
||||
# The categories are separated into causes and consequences. Rate limiting is nearly always
|
||||
# downstream — retries against a hostname with no DNS record exhaust the allowance, which then
|
||||
# fails renewals for domains that have nothing wrong with them.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# cert_triage.sh summary — categories, affected domains, and the causal reading
|
||||
# cert_triage.sh --json the same as JSON, for the Certs tab
|
||||
# cert_triage.sh --files=N override how many rotated logs to read
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# CERT_TRIAGE_FILES rotated logs to read, newest first
|
||||
# CERT_TRIAGE_MAX_BYTES bytes read from the end of each
|
||||
# CERT_TRIAGE_LOG_DIR override the log directory; normally found from the NPM container
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
php "$SCRIPT_DIR/cert_triage.php" "$@"
|
||||
@@ -97,10 +97,6 @@ function vv_uptime_write(array $d): bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
function vv_uptime_pct(int $up, int $total): ?float {
|
||||
return $total > 0 ? round($up / $total * 100, 2) : null;
|
||||
}
|
||||
|
||||
// ── Report ────────────────────────────────────────────────────────────────────
|
||||
// Anything that was not perfect over the last seven days, for the Sunday report. Prints nothing
|
||||
// and exits 0 when every domain was clean — the orchestrator's job is to be quiet on a good week,
|
||||
@@ -157,14 +153,12 @@ if ($status || $events) {
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// Shared by --status, --report and the API. Buckets are keyed by time so "the last N" is a key sort, not an assumption about how many
|
||||
// samples a period should contain — a pass that did not run leaves no bucket rather than a zero.
|
||||
// Shared by --status, --report and the API, and defined once in include/auth.php so the three
|
||||
// callers cannot drift apart on what "the last N" means. Buckets are keyed by time, so it is a key
|
||||
// sort rather than an assumption about how many samples a period should contain — a pass that did
|
||||
// not run leaves no bucket rather than a zero.
|
||||
function vv_uptime_window(array $buckets, int $n): ?float {
|
||||
if (!$buckets) return null;
|
||||
krsort($buckets);
|
||||
$u = $t = 0;
|
||||
foreach (array_slice($buckets, 0, $n, true) as $b) { $u += $b['u'] ?? 0; $t += $b['t'] ?? 0; }
|
||||
return vv_uptime_pct($u, $t);
|
||||
return vv_auth_uptime_window($buckets, $n);
|
||||
}
|
||||
|
||||
// ── One pass ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -66,7 +66,10 @@
|
||||
// actions are reads and are deliberately outside it. See README-unraid.md.
|
||||
//
|
||||
// REQUEST
|
||||
// GET ?action=npm_proxies | npm_certs | lldap_users | lldap_groups | authelia_rules
|
||||
// GET ?action=npm_proxies | npm_certs | npm_stats | npm_uptime | lldap_users | lldap_groups
|
||||
// | authelia_rules
|
||||
// GET ?action=npm_why id
|
||||
// GET ?action=access_check domain, uid, path
|
||||
// POST action=npm_create data=<JSON>
|
||||
// POST action=npm_update id, data=<JSON>
|
||||
// POST action=npm_delete id
|
||||
@@ -103,8 +106,9 @@ require_once dirname(__DIR__) . '/include/auth.php';
|
||||
const VV_AUTH_ACTION_PANEL = [
|
||||
// GET
|
||||
'npm_proxies' => 'proxies', 'npm_certs' => 'proxies', 'npm_stats' => 'proxies', 'npm_uptime' => 'proxies',
|
||||
'npm_why' => 'proxies',
|
||||
'lldap_users' => 'users', 'lldap_groups' => 'users', 'lldap_avatar' => 'users',
|
||||
'authelia_rules' => 'acl',
|
||||
'authelia_rules' => 'acl', 'access_check' => 'acl',
|
||||
// POST
|
||||
'npm_create' => 'proxies', 'npm_update' => 'proxies',
|
||||
'npm_delete' => 'proxies', 'npm_toggle' => 'proxies',
|
||||
@@ -115,14 +119,11 @@ const VV_AUTH_ACTION_PANEL = [
|
||||
'authelia_save' => 'acl',
|
||||
];
|
||||
|
||||
// Same windowing rule as uptime_probe.php: buckets are keyed by time, so "the last N" is a key
|
||||
// sort rather than an assumption that every period produced a sample.
|
||||
// The windowing rule lives in include/auth.php and is shared with Tools/uptime_probe.php. It was
|
||||
// three separate copies of the same six lines, which is three places for the definition of "the
|
||||
// last 24 hours" to drift apart while every one of them keeps returning a plausible number.
|
||||
function vv_uptime_window_api(array $buckets, int $n): ?float {
|
||||
if (!$buckets) return null;
|
||||
krsort($buckets);
|
||||
$u = $t = 0;
|
||||
foreach (array_slice($buckets, 0, $n, true) as $b) { $u += $b['u'] ?? 0; $t += $b['t'] ?? 0; }
|
||||
return $t > 0 ? round($u / $t * 100, 2) : null;
|
||||
return vv_auth_uptime_window($buckets, $n);
|
||||
}
|
||||
|
||||
function vv_auth_action_allowed(string $action): bool {
|
||||
@@ -195,6 +196,28 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') {
|
||||
exit;
|
||||
}
|
||||
|
||||
// Why one host is not at 100%. The only read here that goes and looks rather than serving a
|
||||
// stored figure: it opens a socket to the forward target, asks the domain itself, and inspects
|
||||
// the containers. Slow by the standards of this file — several seconds — which is why it is one
|
||||
// host on demand and never part of the list load.
|
||||
//
|
||||
// A read, so it stays in the GET arm with the other reads. It is worth being explicit that this
|
||||
// is safe to leave outside the CSRF guard: every call it makes is a GET, a HEAD, a TCP connect
|
||||
// or a file read, so the worst a forged request achieves is making this machine look at itself.
|
||||
if ($action === 'npm_why') {
|
||||
echo json_encode(vv_npm_why((int) ($_GET['id'] ?? 0)));
|
||||
exit;
|
||||
}
|
||||
|
||||
// Can this user open this URL, and what decided it. Reads NPM, the Authelia instance that this
|
||||
// particular host talks to, and the directory — the three places the answer is split across.
|
||||
if ($action === 'access_check') {
|
||||
echo json_encode(vv_auth_access_check((string) ($_GET['domain'] ?? ''),
|
||||
(string) ($_GET['uid'] ?? ''),
|
||||
(string) ($_GET['path'] ?? '/')));
|
||||
exit;
|
||||
}
|
||||
|
||||
$result = match ($action) {
|
||||
'npm_proxies' => vv_npm_list_proxies(),
|
||||
'npm_certs' => ['ok' => true, 'certs' => vv_npm_list_certs()],
|
||||
|
||||
@@ -308,5 +308,14 @@ if (!file_exists($cacheFile)) {
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Why renewals failed ───────────────────────────────────────────────────────
|
||||
// Reads certbot's own logs and names the categories the failures fall into. A read, so GET: it
|
||||
// opens files and nothing else. Bounded inside vv_cert_triage() by file count and bytes per file,
|
||||
// because the log directory here is 639 MB and a page request must not depend on its size.
|
||||
if ($action === 'triage') {
|
||||
echo json_encode(vv_cert_triage());
|
||||
exit;
|
||||
}
|
||||
|
||||
$data = json_decode(file_get_contents($cacheFile), true) ?: [];
|
||||
echo json_encode(array_merge(['ok' => true], $data));
|
||||
|
||||
@@ -92,6 +92,13 @@ const VV_AI_FINDING_KINDS = [
|
||||
'container_fault' => 'a container is logging a fault about its own environment',
|
||||
'media_misfiled' => 'a series is shelved somewhere its own metadata does not support',
|
||||
'watchdog_strike' => 'a watchdog has counted something far enough to be worth a record',
|
||||
// The auth stack's two. Neither is a conf problem — one is a service behind a proxy host that
|
||||
// has stopped answering, the other is a hostname put behind Authelia that Authelia then lets
|
||||
// everyone past. They are recorded here rather than only drawn on the Auth tab because both
|
||||
// are conditions that persist for weeks without anyone opening that tab: one host on this
|
||||
// installation has served nothing but 5xx for months.
|
||||
'proxy_down' => 'a proxy host has been failing long enough that it is not a blip',
|
||||
'access_open' => 'a hostname behind an auth_request block is not actually protected by any rule',
|
||||
];
|
||||
|
||||
// Which kinds are a statement about Varaverk's configuration, and which are a statement about
|
||||
@@ -205,6 +212,13 @@ function vv_ai_finding_severity(array $f): string {
|
||||
'missing_value' => 'error', // configured to use something that was never supplied
|
||||
'unreachable' => 'warn', // may be transient; the strike system is what escalates it
|
||||
'unknown_target' => 'warn',
|
||||
// A hostname deliberately put behind authentication that authenticates nobody is the one
|
||||
// finding here that is worse the longer it goes unnoticed, and it is never transient —
|
||||
// it is a state of the configuration, not a passing failure.
|
||||
'access_open' => 'error',
|
||||
// The sweep only files this after the outage has already outlasted a restart, so it is
|
||||
// past the point where the strike system would still be deciding.
|
||||
'proxy_down' => 'error',
|
||||
default => 'warn',
|
||||
};
|
||||
}
|
||||
|
||||
@@ -61,6 +61,9 @@
|
||||
// vv_auth_last_transport()
|
||||
// NPM vv_npm_list_proxies(), vv_npm_list_certs(), vv_npm_create_proxy(),
|
||||
// vv_npm_update_proxy(), vv_npm_delete_proxy(), vv_npm_toggle_proxy()
|
||||
// Diagnosis vv_npm_why(), vv_npm_why_findings(), vv_auth_uptime_window(), vv_auth_db_file(),
|
||||
// vv_auth_tcp_probe(), vv_auth_http_probe(), vv_auth_container_for()
|
||||
// — read-only. The only group here that changes nothing.
|
||||
// LLDAP vv_lldap_list_users(), vv_lldap_list_groups(), vv_lldap_create_user(),
|
||||
// vv_lldap_update_user(), vv_lldap_delete_user(), vv_lldap_set_password(),
|
||||
// vv_lldap_create_group(), vv_lldap_delete_group(),
|
||||
@@ -315,6 +318,398 @@ function vv_npm_toggle_proxy(int $id, bool $enabled): array {
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
// ── Why is this host not at 100% ──────────────────────────────────────────────
|
||||
|
||||
// The one implementation of the uptime windowing rule. Buckets are keyed by time, so "the last N"
|
||||
// is a key sort rather than an assumption that every period produced a sample — a pass that did not
|
||||
// run leaves no bucket at all rather than a zero, and averaging over a count would read a probe
|
||||
// outage as a service outage. Tools/uptime_probe.php and api/auth.php both defer to this.
|
||||
function vv_auth_uptime_window(array $buckets, int $n): ?float {
|
||||
if (!$buckets) return null;
|
||||
krsort($buckets);
|
||||
$u = $t = 0;
|
||||
foreach (array_slice($buckets, 0, $n, true) as $b) { $u += $b['u'] ?? 0; $t += $b['t'] ?? 0; }
|
||||
return $t > 0 ? round($u / $t * 100, 2) : null;
|
||||
}
|
||||
|
||||
function vv_auth_db_file(string $name): string {
|
||||
return rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/' . $name;
|
||||
}
|
||||
|
||||
// One TCP connect, timed. The cheapest question that separates "the application is broken" from
|
||||
// "nothing is there at all", and the one the proxy itself cannot answer — NPM reports a 502 for a
|
||||
// refused connection, a closed port and a hung process alike.
|
||||
function vv_auth_tcp_probe(string $host, int $port, int $timeout = 4): array {
|
||||
if ($host === '' || $port <= 0) return ['ok' => false, 'err' => 'no forward target configured'];
|
||||
$t0 = microtime(true);
|
||||
$errno = 0; $errstr = '';
|
||||
// @ because a refused connection and an unresolvable name are both expected answers here, and
|
||||
// a warning raised into the JSON body would corrupt the response this is reported in.
|
||||
$fp = @fsockopen($host, $port, $errno, $errstr, $timeout);
|
||||
$ms = (int) round((microtime(true) - $t0) * 1000);
|
||||
if ($fp === false) return ['ok' => false, 'ms' => $ms, 'err' => $errstr ?: ('errno ' . $errno)];
|
||||
fclose($fp);
|
||||
return ['ok' => true, 'ms' => $ms];
|
||||
}
|
||||
|
||||
// A HEAD against a URL, reporting the same three things for every probe so the caller can compare
|
||||
// the front door and the back door without special-casing either.
|
||||
//
|
||||
// $verify is the whole point of the second call this makes: an identical request that succeeds only
|
||||
// with verification off says the certificate is the fault and the service behind it is fine, which
|
||||
// is otherwise indistinguishable from the site being down.
|
||||
function vv_auth_http_probe(string $url, bool $verify, int $timeout = 5): array {
|
||||
$ch = curl_init($url);
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_NOBODY => true,
|
||||
CURLOPT_FOLLOWLOCATION => false, // a redirect to the auth portal is an answer, not a step
|
||||
CURLOPT_TIMEOUT => $timeout,
|
||||
CURLOPT_CONNECTTIMEOUT => min($timeout, 4),
|
||||
// Marked as the monitor so this cannot land in the access log as a real request and inflate
|
||||
// the very traffic figures shown beside it. Same string npm_access_stats.php drops.
|
||||
CURLOPT_USERAGENT => 'Varaverk-Uptime/1.0',
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_SSL_VERIFYPEER => $verify,
|
||||
CURLOPT_SSL_VERIFYHOST => $verify ? 2 : 0,
|
||||
]);
|
||||
curl_exec($ch);
|
||||
$r = ['code' => (int) curl_getinfo($ch, CURLINFO_HTTP_CODE),
|
||||
'ms' => (int) round(curl_getinfo($ch, CURLINFO_TOTAL_TIME) * 1000),
|
||||
'err' => curl_error($ch)];
|
||||
curl_close($ch);
|
||||
return $r;
|
||||
}
|
||||
|
||||
// Which container is behind a forward target, if any. Matched on the name first and the container
|
||||
// IP second, because both forms are in use here — some hosts forward to a container name on a
|
||||
// custom network and some to an address on br0.
|
||||
//
|
||||
// Returns null rather than guessing. A forward target that is another machine entirely is a normal
|
||||
// configuration, and reporting "no container" for it is correct, not a failure to find one.
|
||||
function vv_auth_container_for(string $fwdHost): ?array {
|
||||
if ($fwdHost === '') return null;
|
||||
require_once __DIR__ . '/docker.php';
|
||||
$all = vv_dk_inspect_all();
|
||||
if (!$all) return null;
|
||||
|
||||
$hit = function (string $name, array $c, string $how): array {
|
||||
$nets = $c['networks'] ?? [];
|
||||
return ['name' => $name, 'running' => $c['running'], 'status' => $c['status'], 'match' => $how,
|
||||
'networks' => $nets, 'ip' => $nets ? reset($nets) : '',
|
||||
'network' => $nets ? (string) array_key_first($nets) : ''];
|
||||
};
|
||||
|
||||
$needle = strtolower($fwdHost);
|
||||
foreach ($all as $name => $c) if (strtolower($name) === $needle) return $hit($name, $c, 'name');
|
||||
foreach ($all as $name => $c)
|
||||
foreach ($c['networks'] ?? [] as $ip) if ($ip === $fwdHost) return $hit($name, $c, 'ip');
|
||||
return null;
|
||||
}
|
||||
|
||||
// Where this process should actually knock, which is not always what the proxy host says.
|
||||
//
|
||||
// Most forward targets here are container names on a user-defined docker network. Those names are
|
||||
// resolved by docker's embedded DNS, which only the containers on that network can see — NPM
|
||||
// resolves NextCloud perfectly and PHP running on the host cannot resolve it at all. A check that
|
||||
// treated its own resolution failure as evidence would report every one of them as dead, which is
|
||||
// the exact false alarm this whole dialog exists to stop someone chasing.
|
||||
//
|
||||
// So an unresolvable name that matches a running container is redirected to that container's
|
||||
// address, and the substitution is reported rather than hidden — the reader needs to know which
|
||||
// address the result below actually describes.
|
||||
function vv_auth_probe_target(string $fwdHost, ?array $container): array {
|
||||
if ($fwdHost === '' || filter_var($fwdHost, FILTER_VALIDATE_IP)) return ['host' => $fwdHost, 'note' => ''];
|
||||
// gethostbyname() hands back its input unchanged when it cannot resolve — the documented way it
|
||||
// fails, and the reason this is a comparison rather than a truthiness test.
|
||||
if (@gethostbyname($fwdHost) !== $fwdHost) return ['host' => $fwdHost, 'note' => ''];
|
||||
|
||||
if ($container && ($container['ip'] ?? '') !== '')
|
||||
return ['host' => $container['ip'],
|
||||
'note' => $fwdHost . ' is a docker name that only resolves on the ' . ($container['network'] ?: 'proxy')
|
||||
. ' network, so this checked the container address ' . $container['ip'] . ' instead.'];
|
||||
|
||||
return ['host' => $fwdHost,
|
||||
'note' => $fwdHost . ' does not resolve from this host and matches no container here, so the direct '
|
||||
. 'check below could not be made. What the proxy itself reports is the reliable part.'];
|
||||
}
|
||||
|
||||
// Everything known about why one proxy host is not at 100%, gathered in one pass.
|
||||
//
|
||||
// The Proxies tab shows an uptime percentage and nothing about what is behind it, and the causes
|
||||
// look identical from the row: the application is down, its container is not running, the
|
||||
// certificate stopped validating, or the proxy reaches the application perfectly and the
|
||||
// application is the thing returning 5xx. Those are four different jobs and the row cannot tell
|
||||
// them apart, so every low figure has so far meant opening NPM, then Docker, then a terminal.
|
||||
//
|
||||
// Three of the four sources are already on disk — the probe history, the access-log totals, NPM's
|
||||
// own record. The fourth is the part no stored figure can answer: what happens right now, asked
|
||||
// separately of the front door and the back. A host that answers on 10.0.0.5:8096 but fails through
|
||||
// https://name/ is a proxy or certificate fault; one that fails both is the service itself.
|
||||
//
|
||||
// Read-only by construction. Every call below is a GET, a HEAD, a TCP connect or a file read —
|
||||
// nothing here restarts, rewrites or retries anything, because the value of a diagnosis is that it
|
||||
// can be run on a host that is limping without being the thing that finishes it off.
|
||||
function vv_npm_why(int $id): array {
|
||||
$p = vv_npm_list_proxies();
|
||||
if (!($p['ok'] ?? false)) return ['ok' => false, 'error' => $p['error'] ?? 'Could not read proxy hosts from NPM'];
|
||||
|
||||
$host = null;
|
||||
foreach ($p['proxies'] as $h) if ((int) ($h['id'] ?? 0) === $id) { $host = $h; break; }
|
||||
if (!$host) return ['ok' => false, 'error' => "No proxy host with id $id — the list may be stale, reload the tab."];
|
||||
|
||||
$adv = trim((string) ($host['advanced_config'] ?? ''));
|
||||
$guarded = str_contains($adv, 'auth_request');
|
||||
$fwdHost = trim((string) ($host['forward_host'] ?? ''));
|
||||
$fwdPort = (int) ($host['forward_port'] ?? 0);
|
||||
$fwdScheme = (string) ($host['forward_scheme'] ?? 'http');
|
||||
$enabled = ($host['enabled'] ?? true) ? true : false;
|
||||
|
||||
// Wildcards are excluded for the same reason the probe excludes them: *.example.com is not a
|
||||
// hostname anything can connect to, so a live check against it would report a fault that only
|
||||
// describes the check.
|
||||
$domains = [];
|
||||
foreach ($host['domain_names'] ?? [] as $d) {
|
||||
$d = strtolower(trim((string) $d));
|
||||
if ($d !== '' && !str_contains($d, '*')) $domains[] = $d;
|
||||
}
|
||||
|
||||
$out = [
|
||||
'ok' => true,
|
||||
'id' => $id,
|
||||
'enabled' => $enabled,
|
||||
'domains' => $host['domain_names'] ?? [],
|
||||
'forward' => $fwdScheme . '://' . $fwdHost . ':' . $fwdPort,
|
||||
'guarded' => $guarded,
|
||||
'checked' => time(),
|
||||
];
|
||||
|
||||
// ── Certificate ──
|
||||
// Expiry is the single most common reason a host that worked for months stops, and the row
|
||||
// cannot show it because the row is about uptime.
|
||||
$cert = null;
|
||||
$certId = (int) ($host['certificate_id'] ?? 0);
|
||||
if ($certId > 0) {
|
||||
foreach (vv_npm_list_certs() as $c) {
|
||||
if ((int) ($c['id'] ?? 0) !== $certId) continue;
|
||||
$exp = strtotime((string) ($c['expires_on'] ?? '')) ?: null;
|
||||
$cert = [
|
||||
'name' => $c['nice_name'] ?? implode(', ', $c['domain_names'] ?? []),
|
||||
'provider' => $c['provider'] ?? '',
|
||||
'expires' => $exp,
|
||||
'days_left' => $exp ? (int) floor(($exp - time()) / 86400) : null,
|
||||
];
|
||||
break;
|
||||
}
|
||||
}
|
||||
$out['cert'] = $cert;
|
||||
|
||||
// ── Recorded history ──
|
||||
$u = is_file(vv_auth_db_file('uptime.json'))
|
||||
? (json_decode((string) @file_get_contents(vv_auth_db_file('uptime.json')), true) ?: []) : [];
|
||||
$recs = [];
|
||||
foreach ($domains as $d) {
|
||||
$r = $u['domains'][$d] ?? null;
|
||||
if (!$r) continue;
|
||||
// The hours that actually lost something, rather than all 48. "Every hour lost two samples"
|
||||
// and "one hour lost forty" are the same daily percentage and completely different faults,
|
||||
// and this is the only place that distinction survives.
|
||||
$bad = [];
|
||||
$hours = $r['hours'] ?? [];
|
||||
krsort($hours);
|
||||
foreach (array_slice($hours, 0, 24, true) as $k => $b) {
|
||||
$t = $b['t'] ?? 0; $up = $b['u'] ?? 0;
|
||||
if ($t > 0 && $up < $t) $bad[] = ['hour' => $k, 'up' => $up, 'total' => $t];
|
||||
}
|
||||
$recs[$d] = [
|
||||
'state' => $r['state'] ?? null,
|
||||
'h1' => vv_auth_uptime_window($r['hours'] ?? [], 1),
|
||||
'h24' => vv_auth_uptime_window($r['hours'] ?? [], 24),
|
||||
'd30' => vv_auth_uptime_window($r['days'] ?? [], 30),
|
||||
'last_code' => $r['last_code'] ?? null,
|
||||
'last_detail' => $r['last_detail'] ?? null,
|
||||
'last_change' => $r['last_change'] ?? null,
|
||||
'last_ms' => $r['last_ms'] ?? null,
|
||||
'checks' => $r['checks'] ?? 0,
|
||||
// Newest first — a flap is read backwards from now, not forwards from whenever the
|
||||
// record happens to start.
|
||||
'events' => array_slice(array_reverse($r['events'] ?? []), 0, 8),
|
||||
'bad_hours' => $bad,
|
||||
];
|
||||
}
|
||||
$out['history'] = $recs;
|
||||
|
||||
// ── Access-log totals ──
|
||||
$a = is_file(vv_auth_db_file('npm_access.json'))
|
||||
? (json_decode((string) @file_get_contents(vv_auth_db_file('npm_access.json')), true) ?: []) : [];
|
||||
$out['traffic'] = $a['hosts'][(string) $id] ?? null;
|
||||
|
||||
// ── Live, right now ──
|
||||
$cont = vv_auth_container_for($fwdHost);
|
||||
$target = vv_auth_probe_target($fwdHost, $cont);
|
||||
$out['upstream'] = [
|
||||
'host' => $fwdHost,
|
||||
'port' => $fwdPort,
|
||||
'probed' => $target['host'],
|
||||
'note' => $target['note'],
|
||||
'container' => $cont,
|
||||
'tcp' => vv_auth_tcp_probe($target['host'], $fwdPort),
|
||||
];
|
||||
// Only worth asking once something is listening — an HTTP probe of a closed port re-reports the
|
||||
// TCP failure in a less specific form.
|
||||
if ($out['upstream']['tcp']['ok'] ?? false) {
|
||||
// Verification off deliberately: this is an internal hop to an address on this machine's own
|
||||
// network, usually plain HTTP and usually a self-signed certificate when it is not. The
|
||||
// question here is whether the application answers, and the certificate question is asked
|
||||
// at the front door where it actually applies.
|
||||
$out['upstream']['http'] = vv_auth_http_probe($fwdScheme . '://' . $target['host'] . ':' . $fwdPort . '/', false);
|
||||
}
|
||||
|
||||
// Bounded. A host carrying a dozen names would otherwise turn one button press into a dozen
|
||||
// sequential TLS handshakes, and the first few answer the question.
|
||||
$out['live'] = [];
|
||||
foreach (array_slice($domains, 0, 4) as $d) {
|
||||
$r = vv_auth_http_probe('https://' . $d . '/', true);
|
||||
// The second call is the diagnosis, not a retry: succeeding here after failing above is
|
||||
// what proves the certificate rather than the service.
|
||||
if ($r['err'] !== '' && preg_match('/certificat|SSL|TLS/i', $r['err'])) {
|
||||
$r['insecure'] = vv_auth_http_probe('https://' . $d . '/', false);
|
||||
}
|
||||
$out['live'][$d] = $r;
|
||||
}
|
||||
|
||||
$out['findings'] = vv_npm_why_findings($out);
|
||||
return $out;
|
||||
}
|
||||
|
||||
// The deterministic half of the answer, written as sentences rather than codes.
|
||||
//
|
||||
// Separate from the gathering so it can be read, argued with and corrected on its own — and so the
|
||||
// dialog has something certain to show whether or not there is a model on this node to interpret
|
||||
// it. Most low figures on this installation have one of these causes, and none of them needs a
|
||||
// language model to reach.
|
||||
//
|
||||
// Ordered most decisive first: the caller shows them in order and the first line is meant to be the
|
||||
// answer. Each entry is ['level' => bad|warn|info, 'text' => …].
|
||||
function vv_npm_why_findings(array $w): array {
|
||||
$f = [];
|
||||
$up = $w['upstream'] ?? [];
|
||||
$tcp = $up['tcp'] ?? [];
|
||||
$cont = $up['container'] ?? null;
|
||||
|
||||
// Whether the front door is failing *now*, decided once. Several readings below change meaning
|
||||
// entirely on it — "the application answers directly" is a useful clue during an outage and a
|
||||
// false alarm when the site is simply working, and an earlier draft said the second as if it
|
||||
// were the first.
|
||||
$frontBad = false;
|
||||
foreach ($w['live'] ?? [] as $r)
|
||||
if (($r['code'] ?? 0) === 0 || ($r['code'] ?? 0) >= 500) $frontBad = true;
|
||||
|
||||
if (!($w['enabled'] ?? true))
|
||||
$f[] = ['level' => 'bad', 'text' => 'This host is disabled in NPM, so nothing is being served for it. The probe still counts it as unreachable.'];
|
||||
|
||||
if ($cont && !($cont['running'] ?? true))
|
||||
$f[] = ['level' => 'bad', 'text' => 'Container ' . $cont['name'] . ' is ' . ($cont['status'] ?? 'not running')
|
||||
. ' — nothing can answer on ' . ($up['host'] ?? '') . ':' . ($up['port'] ?? '') . '.'];
|
||||
|
||||
// The direct check, stated as what it is. Whether it could be made at all is reported first,
|
||||
// because a check that did not happen must never be read as a check that failed — see
|
||||
// vv_auth_probe_target(). Most forward targets on this machine are docker names this process
|
||||
// cannot resolve, and an earlier draft of this reported every one of them as a dead service.
|
||||
$reached = ($up['probed'] ?? '') !== '' && (($tcp['ok'] ?? false) || !str_contains((string) ($tcp['err'] ?? ''), 'getaddrinfo'));
|
||||
if (!$reached) {
|
||||
$f[] = ['level' => 'info', 'text' => ($up['note'] ?: 'The forward target could not be resolved from here, so no direct check was made.')
|
||||
. ' Nothing below is evidence that the service is down.'];
|
||||
} elseif (!($tcp['ok'] ?? false)) {
|
||||
$f[] = ['level' => 'bad', 'text' => 'Nothing is listening on ' . ($up['probed'] ?? '') . ':' . ($up['port'] ?? '')
|
||||
. ' — ' . ($tcp['err'] ?? 'no reason given')
|
||||
. ($cont && ($cont['running'] ?? false)
|
||||
? '. Container ' . $cont['name'] . ' is running, so the container is up and the application inside it is not serving that port.'
|
||||
: '. The proxy has nothing to forward to.')];
|
||||
} else {
|
||||
$uh = $up['http'] ?? null;
|
||||
if ($uh && $uh['code'] >= 500)
|
||||
$f[] = ['level' => 'bad', 'text' => 'The service is listening on ' . ($up['probed'] ?? '') . ':' . ($up['port'] ?? '')
|
||||
. ' and answered HTTP ' . $uh['code'] . ' itself. The proxy is forwarding correctly — the fault is inside the application.'];
|
||||
elseif ($uh && $uh['code'] === 0 && ($uh['err'] ?? '') !== '')
|
||||
$f[] = ['level' => 'warn', 'text' => 'The port on ' . ($up['probed'] ?? '') . ' is open but nothing came back over it (' . $uh['err']
|
||||
. '). Something is holding the socket without serving — which is what the proxy sees as a timeout.'];
|
||||
elseif ($uh && $frontBad)
|
||||
$f[] = ['level' => 'info', 'text' => 'The service answers directly on ' . ($up['probed'] ?? '') . ':' . ($up['port'] ?? '')
|
||||
. ' with HTTP ' . $uh['code'] . ', so whatever is failing sits between the proxy and it, not in the application.'];
|
||||
}
|
||||
|
||||
// The certificate, from both directions: what NPM says about its expiry, and what a live
|
||||
// handshake actually did. Either can be the fault on its own — a cert with weeks left still
|
||||
// fails if the chain it is serving is wrong.
|
||||
$c = $w['cert'] ?? null;
|
||||
if ($c && $c['days_left'] !== null) {
|
||||
if ($c['days_left'] < 0)
|
||||
$f[] = ['level' => 'bad', 'text' => 'The certificate expired ' . abs($c['days_left']) . ' days ago. Every HTTPS request to this host fails verification.'];
|
||||
elseif ($c['days_left'] <= 14)
|
||||
$f[] = ['level' => 'warn', 'text' => 'The certificate expires in ' . $c['days_left'] . ' days — check the Certs tab for whether renewal is running.'];
|
||||
}
|
||||
|
||||
foreach ($w['live'] ?? [] as $dom => $r) {
|
||||
if (isset($r['insecure']) && $r['insecure']['code'] > 0 && $r['insecure']['code'] < 500) {
|
||||
$f[] = ['level' => 'bad', 'text' => $dom . ' answers normally when certificate verification is turned off. '
|
||||
. 'The service is up and the certificate is what is failing: ' . $r['err']];
|
||||
} elseif ($r['code'] === 0 && ($r['err'] ?? '') !== '') {
|
||||
$f[] = ['level' => 'bad', 'text' => $dom . ' did not answer just now — ' . $r['err']];
|
||||
} elseif ($r['code'] === 502 || $r['code'] === 504) {
|
||||
// NPM's own verdict, and the one piece of evidence that is always authoritative: it is
|
||||
// the component that actually has to reach the upstream. Paired with a direct probe
|
||||
// that succeeded, it stops being "the app is down" and becomes a routing problem —
|
||||
// the proxy and the application are on networks that cannot see each other.
|
||||
$ok = ($up['http']['code'] ?? 0) > 0 && ($up['http']['code'] ?? 0) < 500;
|
||||
$f[] = ['level' => 'bad', 'text' => $dom . ' answered HTTP ' . $r['code'] . ' through the proxy — NPM could not reach '
|
||||
. ($up['host'] ?? '') . ':' . ($up['port'] ?? '') . '.'
|
||||
. ($ok ? ' It answers fine when asked directly, so the two are not on a network that can see each other,'
|
||||
. ' or the forward host is written in a form NPM cannot resolve.' : '')];
|
||||
} elseif ($r['code'] >= 500) {
|
||||
$f[] = ['level' => 'bad', 'text' => $dom . ' answered HTTP ' . $r['code'] . ' through the proxy.'];
|
||||
}
|
||||
}
|
||||
|
||||
// Traffic, read against whether the host is guarded. An unguarded host that is nothing but 4xx
|
||||
// is broken; a guarded one that is nothing but 4xx is usually Authelia doing its job to
|
||||
// unauthenticated callers, and calling that a fault would send someone to fix what is working.
|
||||
$t = $w['traffic'] ?? null;
|
||||
if ($t && ($t['requests'] ?? 0) > 0) {
|
||||
$req = (int) $t['requests'];
|
||||
$s5 = (int) ($t['s5xx'] ?? 0);
|
||||
$s4 = (int) ($t['s4xx'] ?? 0);
|
||||
if ($s5 / $req >= 0.5)
|
||||
$f[] = ['level' => 'bad', 'text' => round($s5 / $req * 100) . '% of all logged requests to this host are 5xx — this has been failing for real users, not just the probe.'];
|
||||
if ($s4 / $req >= 0.9)
|
||||
$f[] = $w['guarded']
|
||||
? ['level' => 'info', 'text' => round($s4 / $req * 100) . '% of requests are 4xx, which on a host behind auth_request is usually Authelia refusing unauthenticated callers rather than a fault.']
|
||||
: ['level' => 'warn', 'text' => round($s4 / $req * 100) . '% of requests are 4xx and nothing is guarding this host, so callers are being refused by the application itself.'];
|
||||
}
|
||||
|
||||
// Shape of the loss. Same percentage, two entirely different problems, and this is the only
|
||||
// reading that separates them.
|
||||
foreach ($w['history'] ?? [] as $dom => $h) {
|
||||
$ev = $h['events'] ?? [];
|
||||
// Over what period, not just how many. Four state changes is flapping if they were this
|
||||
// afternoon and completely unremarkable if they were spread across a month, and a count on
|
||||
// its own cannot tell those apart — the store keeps the last twenty however old they are.
|
||||
$span = count($ev) >= 2 ? ((int) ($ev[0]['ts'] ?? 0) - (int) ($ev[count($ev) - 1]['ts'] ?? 0)) : 0;
|
||||
$spanTxt = $span >= 172800 ? 'over ' . round($span / 86400) . ' days'
|
||||
: ($span >= 7200 ? 'over ' . round($span / 3600) . ' hours' : 'within the hour');
|
||||
if (count($ev) >= 4)
|
||||
$f[] = ['level' => 'warn', 'text' => $dom . ' changed state ' . count($ev) . ' times ' . $spanTxt
|
||||
. ' — this is flapping rather than one clean outage, so look for something restarting on a cycle.'];
|
||||
elseif (count($h['bad_hours'] ?? []) === 1 && ($h['state'] ?? '') === 'up')
|
||||
$f[] = ['level' => 'info', 'text' => $dom . ' lost samples in one hour only (' . $h['bad_hours'][0]['up'] . '/' . $h['bad_hours'][0]['total']
|
||||
. ' at ' . substr((string) $h['bad_hours'][0]['hour'], 8, 2) . ':00) and has been up since. A single event, already over.'];
|
||||
}
|
||||
|
||||
if (!$f)
|
||||
$f[] = ['level' => 'info', 'text' => 'Nothing is failing right now — the service answers, the certificate verifies and the proxy is forwarding. Whatever cost this host its uptime is in the history below and has already ended.'];
|
||||
|
||||
return $f;
|
||||
}
|
||||
|
||||
// ── lldap ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_lldap_token(): string {
|
||||
@@ -569,9 +964,14 @@ function vv_lldap_remove_from_group(string $userId, int $groupId): array {
|
||||
|
||||
// ── Authelia ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_authelia_read_rules(): array {
|
||||
// $file overrides the configured path. There is more than one Authelia on this machine — the
|
||||
// primary serves the .com names and Authelia-Secondary serves the .us ones — and conf names only
|
||||
// the primary, so anything reasoning about a specific domain has to be able to read the instance
|
||||
// that domain actually talks to. Defaults to the configured one, so every existing caller and the
|
||||
// whole editing path are unchanged.
|
||||
function vv_authelia_read_rules(?string $file = null): array {
|
||||
$conf = vv_auth_conf();
|
||||
$file = $conf['authelia_config'];
|
||||
$file = $file ?: $conf['authelia_config'];
|
||||
if (!file_exists($file)) return ['ok' => false, 'error' => 'Config not found: ' . $file];
|
||||
|
||||
$content = file_get_contents($file);
|
||||
@@ -791,3 +1191,475 @@ function vv_authelia_yaml_scalar(string $val): string {
|
||||
return '"' . str_replace(['\\', '"'], ['\\\\', '\\"'], $val) . '"';
|
||||
return $val;
|
||||
}
|
||||
|
||||
// ── Access simulation ─────────────────────────────────────────────────────────
|
||||
//
|
||||
// Whether one person can open one URL is decided by three objects that no single card shows: the
|
||||
// NPM host (does it hand the request to Authelia at all), the Authelia rule list (which rule wins,
|
||||
// in file order), and the LDAP group membership (does the winning rule's subject include them).
|
||||
// Any one of the three can be the reason a login is refused — or worse, not asked for — and the
|
||||
// only way to find out has been to read three configs and reason about them by hand.
|
||||
//
|
||||
// This walks it the way Authelia does and reports every step, so the answer is checkable rather
|
||||
// than asserted. It changes nothing: every function below reads.
|
||||
|
||||
// Which Authelia a proxy host actually talks to, read out of its own nginx block rather than
|
||||
// assumed from conf.
|
||||
//
|
||||
// This machine runs two — Authelia for the .com names and Authelia-Secondary for the .us ones —
|
||||
// and HOST1_AUTHELIA_CONFIG names only the first. Evaluating a .us domain against the primary's
|
||||
// rules would produce a confident, wrong answer for six live hostnames, so the instance is taken
|
||||
// from the `set $upstream_authelia http://NAME:PORT` line that decides it in production.
|
||||
function vv_authelia_instance_for(array $proxyHost): array {
|
||||
$adv = (string) ($proxyHost['advanced_config'] ?? '');
|
||||
if (!str_contains($adv, 'auth_request'))
|
||||
return ['guarded' => false, 'container' => '', 'config' => '', 'source' => 'none'];
|
||||
|
||||
$container = '';
|
||||
if (preg_match('#set\s+\$upstream_authelia\s+https?://([A-Za-z0-9._-]+):(\d+)#', $adv, $m))
|
||||
$container = $m[1];
|
||||
|
||||
$conf = vv_auth_conf();
|
||||
// The configured instance is matched by name rather than assumed, so the tab's own editing
|
||||
// target is identified as such and anything else is reported as the separate instance it is.
|
||||
if ($container !== '' && strcasecmp($container, (string) ($conf['authelia_container'] ?? '')) === 0)
|
||||
return ['guarded' => true, 'container' => $container, 'config' => $conf['authelia_config'],
|
||||
'source' => 'conf', 'is_configured' => true];
|
||||
|
||||
$path = $container !== '' ? vv_authelia_config_for_container($container) : '';
|
||||
return ['guarded' => true, 'container' => $container, 'config' => $path,
|
||||
'source' => $path !== '' ? 'docker' : 'unknown', 'is_configured' => false];
|
||||
}
|
||||
|
||||
// A container's configuration.yml, found through its own /config bind mount. Nothing hardcodes a
|
||||
// path: a second instance added later is picked up because it is mounted the same way, which is the
|
||||
// same reason the rest of this plugin reads its host list from NPM rather than from conf.
|
||||
function vv_authelia_config_for_container(string $name): string {
|
||||
static $cache = [];
|
||||
if (isset($cache[$name])) return $cache[$name];
|
||||
$cache[$name] = '';
|
||||
|
||||
require_once __DIR__ . '/docker.php';
|
||||
$all = vv_dk_inspect_all();
|
||||
foreach ($all as $cn => $c) {
|
||||
if (strcasecmp($cn, $name) !== 0) continue;
|
||||
foreach ($c['mounts'] ?? [] as $m) {
|
||||
if (($m['dst'] ?? '') !== '/config') continue;
|
||||
$p = rtrim((string) $m['src'], '/') . '/configuration.yml';
|
||||
if (is_file($p)) $cache[$name] = $p;
|
||||
}
|
||||
}
|
||||
return $cache[$name];
|
||||
}
|
||||
|
||||
// Does an Authelia domain pattern match this hostname? Authelia accepts an exact name and a single
|
||||
// leading wildcard label; nothing else here uses regex domains, so nothing else is claimed.
|
||||
function vv_authelia_domain_matches(string $pattern, string $domain): bool {
|
||||
$pattern = strtolower(trim($pattern));
|
||||
$domain = strtolower(trim($domain));
|
||||
if ($pattern === '' || $domain === '') return false;
|
||||
if ($pattern === $domain) return true;
|
||||
if (str_starts_with($pattern, '*.')) {
|
||||
$suffix = substr($pattern, 1); // ".example.com"
|
||||
// One label only, matching Authelia: *.example.com covers a.example.com and not a.b.example.com.
|
||||
return str_ends_with($domain, $suffix)
|
||||
&& !str_contains(substr($domain, 0, -strlen($suffix)), '.')
|
||||
&& substr($domain, 0, -strlen($suffix)) !== '';
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Whether a rule's subject admits this user. Authelia's shape is a list of subjects OR'd together,
|
||||
// where an element that is itself a list is AND'd — so [[a,b],c] means "(a and b) or c".
|
||||
//
|
||||
// A rule with no subject at all applies to everyone, which is the case that silently shadows every
|
||||
// specific rule below it. Returned as a reason string as well as a verdict, because "the rule was
|
||||
// skipped" and "the rule matched and denied" look identical in a result and mean opposite things.
|
||||
function vv_authelia_subject_matches($subject, string $uid, array $groups): array {
|
||||
if ($subject === null || $subject === '' || $subject === [])
|
||||
return ['match' => true, 'why' => 'no subject — applies to everyone'];
|
||||
|
||||
$groupsLc = array_map('strtolower', $groups);
|
||||
$one = function (string $s) use ($uid, $groupsLc): bool {
|
||||
$s = trim($s);
|
||||
if (str_starts_with($s, 'group:')) return in_array(strtolower(substr($s, 6)), $groupsLc, true);
|
||||
if (str_starts_with($s, 'user:')) return strcasecmp(substr($s, 5), $uid) === 0;
|
||||
// An unprefixed subject is a username in Authelia's schema.
|
||||
return strcasecmp($s, $uid) === 0;
|
||||
};
|
||||
|
||||
$alternatives = is_array($subject) ? $subject : [$subject];
|
||||
foreach ($alternatives as $alt) {
|
||||
if (is_array($alt)) {
|
||||
$all = true;
|
||||
foreach ($alt as $part) if (!$one((string) $part)) { $all = false; break; }
|
||||
if ($all) return ['match' => true, 'why' => 'matches all of ' . implode(' + ', array_map('strval', $alt))];
|
||||
} elseif ($one((string) $alt)) {
|
||||
return ['match' => true, 'why' => 'matches ' . $alt];
|
||||
}
|
||||
}
|
||||
$flat = [];
|
||||
foreach ($alternatives as $alt) $flat[] = is_array($alt) ? '(' . implode(' + ', array_map('strval', $alt)) . ')' : (string) $alt;
|
||||
return ['match' => false, 'why' => 'not ' . implode(' or ', $flat)];
|
||||
}
|
||||
|
||||
// Walk the rules in file order and stop at the first that matches on every axis, which is exactly
|
||||
// what Authelia does. Every rule considered is reported with why it did or did not apply — the
|
||||
// trace is the point, because "which rule won" is rarely the surprising part. "Which rule you
|
||||
// thought would win and why it was skipped" is.
|
||||
function vv_authelia_evaluate(array $rules, string $defaultPolicy, string $domain, string $path,
|
||||
string $uid, array $groups): array {
|
||||
$trace = [];
|
||||
foreach ($rules as $i => $r) {
|
||||
$doms = is_array($r['domain'] ?? '') ? $r['domain'] : [$r['domain'] ?? ''];
|
||||
$domHit = false;
|
||||
foreach ($doms as $d) if (vv_authelia_domain_matches((string) $d, $domain)) { $domHit = true; break; }
|
||||
$label = trim(preg_replace('/^#+\s*/', '', implode(' ', $r['_label'] ?? []))) ?: ('rule ' . ($i + 1));
|
||||
|
||||
if (!$domHit) { $trace[] = ['n' => $i + 1, 'label' => $label, 'applied' => false, 'skip' => 'domain', 'why' => 'domain not listed']; continue; }
|
||||
|
||||
// Resources is a path regex. A rule carrying one only applies to the paths it names, so a
|
||||
// rule that looks like it covers a host may cover one directory of it.
|
||||
$res = $r['resources'] ?? null;
|
||||
if ($res !== null && $res !== '' && $res !== []) {
|
||||
$list = is_array($res) ? $res : [$res];
|
||||
$hit = false;
|
||||
foreach ($list as $rx) {
|
||||
// Delimited and error-suppressed: this pattern comes from a hand-edited file, and a
|
||||
// malformed one must report as "did not match" rather than raising a warning into
|
||||
// the answer.
|
||||
if (@preg_match('#' . str_replace('#', '\#', (string) $rx) . '#', $path)) { $hit = true; break; }
|
||||
}
|
||||
if (!$hit) {
|
||||
$trace[] = ['n' => $i + 1, 'label' => $label, 'applied' => false, 'skip' => 'path',
|
||||
'why' => 'domain matches but the path ' . $path . ' is outside its resources pattern'];
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
$sub = vv_authelia_subject_matches($r['subject'] ?? null, $uid, $groups);
|
||||
if (!$sub['match']) {
|
||||
// Recorded as a subject skip specifically. "No rule mentioned this host" and "a rule
|
||||
// for this host stepped over this person" both end at the default policy and are
|
||||
// completely different facts about the configuration.
|
||||
$trace[] = ['n' => $i + 1, 'label' => $label, 'applied' => false, 'skip' => 'subject',
|
||||
'policy' => $r['policy'] ?? 'deny',
|
||||
'why' => 'domain matches but the user is ' . $sub['why']];
|
||||
continue;
|
||||
}
|
||||
|
||||
$trace[] = ['n' => $i + 1, 'label' => $label, 'applied' => true,
|
||||
'why' => 'domain matches and the user ' . $sub['why'],
|
||||
'policy' => $r['policy'] ?? 'deny'];
|
||||
return ['policy' => $r['policy'] ?? 'deny', 'matched' => $i + 1, 'matched_label' => $label, 'trace' => $trace];
|
||||
}
|
||||
return ['policy' => $defaultPolicy, 'matched' => null, 'matched_label' => '', 'trace' => $trace];
|
||||
}
|
||||
|
||||
// The whole question, end to end: can this user open this URL, and what decided it.
|
||||
function vv_auth_access_check(string $domain, string $uid, string $path = '/'): array {
|
||||
$domain = strtolower(trim($domain));
|
||||
$path = $path === '' ? '/' : $path;
|
||||
if ($domain === '') return ['ok' => false, 'error' => 'No domain given'];
|
||||
|
||||
$p = vv_npm_list_proxies();
|
||||
if (!($p['ok'] ?? false)) return ['ok' => false, 'error' => $p['error'] ?? 'Could not read proxy hosts'];
|
||||
|
||||
$host = null;
|
||||
foreach ($p['proxies'] as $h)
|
||||
foreach ($h['domain_names'] ?? [] as $d)
|
||||
if (vv_authelia_domain_matches((string) $d, $domain) || strtolower((string) $d) === $domain) { $host = $h; break 2; }
|
||||
|
||||
$out = ['ok' => true, 'domain' => $domain, 'path' => $path, 'uid' => $uid, 'findings' => []];
|
||||
|
||||
if (!$host) {
|
||||
$out['findings'][] = ['level' => 'warn', 'text' => 'No NPM proxy host serves ' . $domain
|
||||
. ', so nothing reaches Authelia for it and no rule about it has any effect.'];
|
||||
$out['served'] = false;
|
||||
return $out;
|
||||
}
|
||||
$out['served'] = true;
|
||||
$out['enabled'] = ($host['enabled'] ?? true) ? true : false;
|
||||
$out['forward'] = ($host['forward_scheme'] ?? 'http') . '://' . ($host['forward_host'] ?? '') . ':' . ($host['forward_port'] ?? '');
|
||||
|
||||
// The user's groups, which is the half of the answer that lives in a different system entirely.
|
||||
// Taken from the user record rather than by walking every group, because that record carries
|
||||
// both facts this needs — whether the person exists and what they belong to. Asking the group
|
||||
// list instead would make "in no groups" and "no such person" the same empty result.
|
||||
$groups = [];
|
||||
$known = false;
|
||||
if ($uid !== '') {
|
||||
$ul = vv_lldap_list_users();
|
||||
foreach (($ul['ok'] ?? false) ? ($ul['users'] ?? []) : [] as $u) {
|
||||
if (strcasecmp((string) ($u['id'] ?? ''), $uid) !== 0) continue;
|
||||
$known = true;
|
||||
$groups = array_values(array_filter(array_column($u['groups'] ?? [], 'displayName')));
|
||||
break;
|
||||
}
|
||||
}
|
||||
$out['groups'] = $groups;
|
||||
$out['user_known'] = $known;
|
||||
|
||||
$inst = vv_authelia_instance_for($host);
|
||||
$out['authelia'] = $inst;
|
||||
|
||||
if (!$inst['guarded']) {
|
||||
$out['policy'] = 'bypass';
|
||||
$out['findings'][] = ['level' => 'warn', 'text' => 'This host has no auth_request block, so the request never reaches Authelia. '
|
||||
. 'Anyone who can resolve ' . $domain . ' gets through to the application, whatever the rules say.'];
|
||||
return $out;
|
||||
}
|
||||
if ($inst['config'] === '' || !is_file($inst['config'])) {
|
||||
$out['findings'][] = ['level' => 'warn', 'text' => 'This host sends its authentication to ' . ($inst['container'] ?: 'an unnamed instance')
|
||||
. ', whose configuration could not be located, so the decision below cannot be worked out.'];
|
||||
return $out;
|
||||
}
|
||||
|
||||
$r = vv_authelia_read_rules($inst['config']);
|
||||
if (!($r['ok'] ?? false)) { $out['findings'][] = ['level' => 'warn', 'text' => $r['error'] ?? 'Rules unreadable']; return $out; }
|
||||
|
||||
$ev = vv_authelia_evaluate($r['rules'] ?? [], $r['default_policy'] ?? 'deny', $domain, $path, $uid, $groups);
|
||||
$out['policy'] = $ev['policy'];
|
||||
$out['matched'] = $ev['matched'];
|
||||
$out['matched_label'] = $ev['matched_label'];
|
||||
$out['trace'] = $ev['trace'];
|
||||
$out['default_policy'] = $r['default_policy'] ?? 'deny';
|
||||
$out['findings'] = array_merge($out['findings'], vv_auth_access_findings($out, $inst));
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_auth_access_findings(array $o, array $inst): array {
|
||||
$f = [];
|
||||
|
||||
// The one that cannot be seen from any single page. A rule list that is not the one being
|
||||
// edited on this tab is a rule list nobody is maintaining on purpose.
|
||||
if (!($inst['is_configured'] ?? false))
|
||||
$f[] = ['level' => 'info', 'text' => 'Decided by ' . ($inst['container'] ?: 'a second instance')
|
||||
. ', which is not the Authelia this tab edits. Its rules are in ' . ($inst['config'] ?: 'a config that was not found')
|
||||
. ' and nothing on this page changes them.'];
|
||||
|
||||
if (!($o['enabled'] ?? true))
|
||||
$f[] = ['level' => 'warn', 'text' => 'The proxy host is disabled in NPM, so nothing is served here at all right now.'];
|
||||
|
||||
if ($o['uid'] !== '' && !($o['user_known'] ?? false))
|
||||
$f[] = ['level' => 'warn', 'text' => 'No user with the id "' . $o['uid'] . '" exists in the directory, so this is the answer for a name that cannot log in.'];
|
||||
elseif ($o['uid'] !== '' && !$o['groups'])
|
||||
$f[] = ['level' => 'info', 'text' => $o['uid'] . ' is in no groups, so every rule with a group subject skips them.'];
|
||||
|
||||
$policy = $o['policy'] ?? '';
|
||||
if ($o['matched'] === null) {
|
||||
// Two different facts end at the same default policy, and conflating them was the first
|
||||
// version of this: a host no rule mentions, and a host whose rule stepped over this
|
||||
// particular person. The second is the more pointed one — the rule exists, it was written
|
||||
// for this host, and the default let them past it anyway.
|
||||
$stepped = [];
|
||||
foreach ($o['trace'] ?? [] as $t) if (($t['skip'] ?? '') === 'subject') $stepped[] = $t;
|
||||
|
||||
if ($stepped && $policy === 'bypass') {
|
||||
$t = $stepped[0];
|
||||
$f[] = ['level' => 'bad', 'text' => 'Rule ' . $t['n'] . ' (' . $t['label'] . ') covers ' . $o['domain']
|
||||
. ' but does not apply to this user — ' . $t['why'] . '. No later rule matches either, so the default policy takes over, '
|
||||
. 'and the default here is bypass. The rule written to protect this host lets everyone it does not name straight through.'];
|
||||
// Stated because the answer is different for a caller who is not logged in at all, and
|
||||
// an operator reading "bypass" would otherwise reasonably conclude the host is open to
|
||||
// the internet. Authelia treats an anonymous request against a rule carrying a subject
|
||||
// as a potential match and sends them to the portal first; this simulation answers for
|
||||
// someone who has already authenticated as this user.
|
||||
$f[] = ['level' => 'info', 'text' => 'This is the answer for a caller already logged in as ' . ($o['uid'] ?: 'someone')
|
||||
. '. Authelia handles an anonymous caller differently — a rule carrying a subject makes it send them to the login portal first — '
|
||||
. 'so this is an authenticated user reaching something not meant for them, not an open door to the internet.'];
|
||||
} elseif ($policy === 'bypass') {
|
||||
$f[] = ['level' => 'bad', 'text' => 'No rule mentions ' . $o['domain'] . ' at all, so it falls to the default policy, which is bypass — '
|
||||
. 'the request goes to Authelia and Authelia waves it through. This host is behind an auth_request block that never refuses anyone.'];
|
||||
} else {
|
||||
$f[] = ['level' => 'info', 'text' => 'No rule matches ' . $o['domain'] . ', so the default policy of ' . $policy . ' applies.'];
|
||||
}
|
||||
} else {
|
||||
$f[] = ['level' => $policy === 'deny' ? 'warn' : 'info',
|
||||
'text' => 'Rule ' . $o['matched'] . ' (' . $o['matched_label'] . ') is the first one that applies, and its policy is ' . $policy . '.'];
|
||||
}
|
||||
|
||||
if ($policy === 'bypass' || $policy === '')
|
||||
$f[] = ['level' => 'info', 'text' => ($o['uid'] !== '' ? $o['uid'] : 'Anyone') . ' reaches ' . $o['domain']
|
||||
. ' without being asked to authenticate. Whether that is right depends on whether the application behind it has its own login.'];
|
||||
elseif ($policy === 'deny')
|
||||
$f[] = ['level' => 'warn', 'text' => ($o['uid'] !== '' ? $o['uid'] : 'This caller') . ' is refused before reaching the application.'];
|
||||
else
|
||||
$f[] = ['level' => 'info', 'text' => ($o['uid'] !== '' ? $o['uid'] : 'A caller') . ' is asked to log in (' . $policy . ') and then reaches the application.'];
|
||||
|
||||
return $f;
|
||||
}
|
||||
|
||||
// ── Certificate renewal triage ────────────────────────────────────────────────
|
||||
//
|
||||
// Why renewals failed, from certbot's own logs. cert_history.sh counts failures by noticing an
|
||||
// expiry in the past; this reads the reason. See Tools/cert_triage.php for the full note.
|
||||
|
||||
// The categories renewal failures actually fall into here, in the order a reader should meet them:
|
||||
// causes before consequences. Each pattern is anchored on the string certbot itself emits, so a
|
||||
// category matching is evidence rather than inference.
|
||||
//
|
||||
// 'root' marks a cause worth acting on directly. Rate limiting is deliberately not one — it is
|
||||
// what happens after something else has been failing, and treating it as the problem sends people
|
||||
// to wait out a timer instead of fixing the DNS record that burned it.
|
||||
const VV_CERT_TRIAGE_PATTERNS = [
|
||||
['id' => 'no-dns-record', 'root' => true,
|
||||
'rx' => '/DNS problem: NXDOMAIN looking up [A-Z]+ for ([A-Za-z0-9._-]+)/',
|
||||
'what' => 'the hostname has no DNS record at all'],
|
||||
['id' => 'no-a-record', 'root' => true,
|
||||
'rx' => '/no valid A records found for ([A-Za-z0-9._-]+)/',
|
||||
'what' => 'the hostname resolves but has no address record Let\'s Encrypt can reach'],
|
||||
['id' => 'challenge-unreachable', 'root' => true,
|
||||
'rx' => '/Timeout during connect[^\n]*|Fetching http:\/\/([A-Za-z0-9._-]+)\/\.well-known[^\n]*Timeout/',
|
||||
'what' => 'the HTTP-01 challenge could not be fetched — port 80 is not reaching this proxy'],
|
||||
['id' => 'caa-forbids', 'root' => true,
|
||||
'rx' => '/CAA record for ([A-Za-z0-9._-]+) prevents issuance/',
|
||||
'what' => 'a CAA record on the domain forbids Let\'s Encrypt from issuing'],
|
||||
['id' => 'revoke-expired', 'root' => false,
|
||||
'rx' => '/Unable to revoke :: Certificate is expired/',
|
||||
'what' => 'a revoke was attempted on a certificate that had already expired'],
|
||||
// Last, and marked as a consequence. 2079 of these on this installation, every one of them
|
||||
// downstream of the three hostnames above.
|
||||
['id' => 'rate-limited', 'root' => false,
|
||||
'rx' => '/urn:ietf:params:acme:error:rateLimited/',
|
||||
'what' => 'Let\'s Encrypt refused the request because too many were made too recently'],
|
||||
];
|
||||
|
||||
// The rotated certbot logs, newest first, ordered by their rotation suffix.
|
||||
//
|
||||
// Never by mtime. Every one of these files carries the same mtime on this machine — they live in
|
||||
// the Critical-Data share and are written as a set by the sync, so the filesystem says all
|
||||
// thousand were modified in the same minute. Sorting by mtime picks an arbitrary sample from
|
||||
// anywhere in the history and presents it as the current state.
|
||||
function vv_cert_log_files(int $limit): array {
|
||||
$dir = trim((string) (vv_conf_vars()['CERT_TRIAGE_LOG_DIR'] ?? ''));
|
||||
if ($dir === '') {
|
||||
require_once __DIR__ . '/docker.php';
|
||||
foreach (vv_dk_inspect_all() as $name => $c) {
|
||||
if (stripos($name, 'nginx') === false && stripos($name, 'npm') === false) continue;
|
||||
foreach ($c['mounts'] ?? [] as $m) {
|
||||
if (($m['dst'] ?? '') !== '/config' && ($m['dst'] ?? '') !== '/data') continue;
|
||||
$p = rtrim((string) $m['src'], '/') . '/log';
|
||||
if (is_dir($p)) { $dir = $p; break 2; }
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($dir === '' || !is_dir($dir)) return ['dir' => $dir, 'files' => []];
|
||||
|
||||
$found = glob($dir . '/letsencrypt.log*') ?: [];
|
||||
$rank = [];
|
||||
foreach ($found as $f) {
|
||||
// letsencrypt.log is the live one and sorts ahead of every numbered rotation.
|
||||
$n = preg_match('/\.log\.(\d+)$/', $f, $m) ? (int) $m[1] : -1;
|
||||
$rank[$f] = $n;
|
||||
}
|
||||
asort($rank);
|
||||
return ['dir' => $dir, 'files' => array_slice(array_keys($rank), 0, max(1, $limit))];
|
||||
}
|
||||
|
||||
// The tail of a file, without reading the whole thing. These run to a megabyte each and the
|
||||
// interesting part of a certbot run is always at the end.
|
||||
function vv_cert_log_tail(string $path, int $maxBytes): string {
|
||||
$size = @filesize($path);
|
||||
if ($size === false) return '';
|
||||
$fh = @fopen($path, 'rb');
|
||||
if (!$fh) return '';
|
||||
if ($size > $maxBytes) @fseek($fh, $size - $maxBytes);
|
||||
$data = (string) @stream_get_contents($fh);
|
||||
fclose($fh);
|
||||
return $data;
|
||||
}
|
||||
|
||||
function vv_cert_triage(int $filesOverride = 0): array {
|
||||
$v = vv_conf_vars();
|
||||
$lim = $filesOverride > 0 ? $filesOverride : max(1, (int) ($v['CERT_TRIAGE_FILES'] ?? 40));
|
||||
$max = max(4096, (int) ($v['CERT_TRIAGE_MAX_BYTES'] ?? 262144));
|
||||
|
||||
$found = vv_cert_log_files($lim);
|
||||
if (!$found['files'])
|
||||
return ['ok' => false, 'error' => 'No certbot logs found'
|
||||
. ($found['dir'] !== '' ? ' in ' . $found['dir'] : ' — set CERT_TRIAGE_LOG_DIR')];
|
||||
|
||||
// Counted per run, not per line. One log file is one certbot invocation, and a single failed
|
||||
// run writes its reason several times over — in the ACME response, in the traceback, and again
|
||||
// in certbot's own ERROR summary. Counting lines therefore reports one failure as three and
|
||||
// makes the categories incomparable with each other, because the noisier reasons repeat more.
|
||||
// "12 of 40 runs failed for no DNS record" is a number that means something.
|
||||
$counts = $doms = [];
|
||||
$runs = $failedRuns = $unclassified = 0;
|
||||
|
||||
foreach ($found['files'] as $f) {
|
||||
$text = vv_cert_log_tail($f, $max);
|
||||
if ($text === '') continue;
|
||||
$runs++;
|
||||
|
||||
$hitAny = false;
|
||||
foreach (VV_CERT_TRIAGE_PATTERNS as $p) {
|
||||
if (!preg_match_all($p['rx'], $text, $m, PREG_SET_ORDER)) continue;
|
||||
$hitAny = true;
|
||||
$counts[$p['id']] = ($counts[$p['id']] ?? 0) + 1;
|
||||
foreach ($m as $hit)
|
||||
// Only some patterns capture a hostname; the others are about the run, not a name.
|
||||
if (isset($hit[1]) && $hit[1] !== '') $doms[$p['id']][strtolower($hit[1])] = true;
|
||||
}
|
||||
if ($hitAny) $failedRuns++;
|
||||
|
||||
// A run that errored and matched nothing known. Reported rather than dropped: an error
|
||||
// certbot starts emitting after this was written has to show up as something, and a total
|
||||
// that quietly shrinks is how a new failure mode stays invisible.
|
||||
if (!$hitAny && preg_match('/:ERROR:certbot/', $text)) { $unclassified++; $failedRuns++; }
|
||||
}
|
||||
|
||||
$cats = [];
|
||||
foreach (VV_CERT_TRIAGE_PATTERNS as $p) {
|
||||
if (empty($counts[$p['id']])) continue;
|
||||
$cats[] = ['id' => $p['id'], 'root' => $p['root'], 'what' => $p['what'],
|
||||
'count' => $counts[$p['id']], 'domains' => array_keys($doms[$p['id']] ?? [])];
|
||||
}
|
||||
|
||||
return ['ok' => true, 'dir' => $found['dir'], 'files_read' => $runs,
|
||||
'total' => $failedRuns, 'unclassified' => $unclassified,
|
||||
'categories' => $cats, 'reading' => vv_cert_triage_reading($cats)];
|
||||
}
|
||||
|
||||
// The causal reading, which is the part a category count cannot give. Written as sentences because
|
||||
// the relationship between these categories is the whole finding: one of them is nearly always
|
||||
// downstream of another, and a list sorted by count puts the consequence at the top.
|
||||
function vv_cert_triage_reading(array $cats): array {
|
||||
if (!$cats) return [];
|
||||
$by = [];
|
||||
foreach ($cats as $c) $by[$c['id']] = $c;
|
||||
|
||||
$roots = array_values(array_filter($cats, fn($c) => $c['root']));
|
||||
$out = [];
|
||||
|
||||
if ($roots) {
|
||||
$names = [];
|
||||
foreach ($roots as $r) foreach ($r['domains'] as $d) $names[$d] = true;
|
||||
$n = count($roots);
|
||||
$out[] = 'Root cause: ' . $n . ' kind' . ($n === 1 ? '' : 's') . ' of failure that '
|
||||
. ($n === 1 ? 'is' : 'are') . ' nobody else\'s consequence'
|
||||
. ($names ? ', affecting ' . implode(', ', array_slice(array_keys($names), 0, 8))
|
||||
. (count($names) > 8 ? ' and ' . (count($names) - 8) . ' more' : '') : '') . '.';
|
||||
}
|
||||
|
||||
if (isset($by['rate-limited'])) {
|
||||
$out[] = $roots
|
||||
? 'The ' . $by['rate-limited']['count'] . ' rate-limit refusals are downstream of that: '
|
||||
. 'certbot retried the failing names often enough to exhaust the allowance, which then '
|
||||
. 'fails renewals for domains that have nothing wrong with them. Fixing the names above '
|
||||
. 'is what clears it — waiting out the limit only restarts the cycle.'
|
||||
: 'Rate limiting is the only category present, with no failing name behind it. That points '
|
||||
. 'at renewal being attempted far too often rather than at any one domain.';
|
||||
}
|
||||
|
||||
if (isset($by['revoke-expired']))
|
||||
$out[] = 'The revoke failures are harmless in themselves — a certificate that already expired '
|
||||
. 'cannot be revoked, and does not need to be. They indicate cleanup running against '
|
||||
. 'certificates that were already dead.';
|
||||
|
||||
if (!$roots && !isset($by['rate-limited']))
|
||||
$out[] = 'Nothing here is a standing cause — these are individual failures rather than a pattern.';
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
@@ -238,8 +238,10 @@ require_once dirname(__DIR__) . '/include/ai_chat.php';
|
||||
|
||||
/* ── Misc ────────────────────────────────────────────────────────────────── */
|
||||
.vv-au-empty { padding:24px;text-align:center;font-size:11px;color:#333; }
|
||||
.vv-au-domain { font-size:12px;color:#bbb;font-weight:bold; }
|
||||
.vv-au-fwd { font-size:10px;color:#444;font-family:monospace;margin-top:1px; }
|
||||
/* The host column is the flexible one in the proxy table, so a long list of domains has to wrap
|
||||
inside it rather than push the figure columns out of line. */
|
||||
.vv-au-domain { font-size:12px;color:#bbb;font-weight:bold;overflow-wrap:anywhere; }
|
||||
.vv-au-fwd { font-size:10px;color:#444;font-family:monospace;margin-top:1px;overflow-wrap:anywhere; }
|
||||
|
||||
/* ── Proxy row marks and traffic ─────────────────────────────────────────── */
|
||||
/* Small marks rather than four more columns. Each one is a fact about the host that had no
|
||||
@@ -253,7 +255,7 @@ require_once dirname(__DIR__) . '/include/ai_chat.php';
|
||||
.vv-au-m.auth { background:#1a0d2a;border-color:#2a1a4a;color:#9c6ff7;font-weight:600; }
|
||||
/* Requests and errors are their own columns now. Right-aligned so the magnitudes line up down the
|
||||
page — the whole point of the errors column is spotting the one host that is all failures. */
|
||||
.vv-au-num { text-align:right;white-space:nowrap;width:1%; }
|
||||
.vv-au-num { text-align:right;white-space:nowrap; }
|
||||
.vv-au-stat-n { font-size:13px;font-weight:700;color:#bbb;line-height:1.15; }
|
||||
.vv-au-stat-n.dim { color:#2e2e2e;font-weight:normal;font-size:12px; }
|
||||
.vv-au-stat-n.ok { color:#2d4a2d; }
|
||||
@@ -270,6 +272,79 @@ require_once dirname(__DIR__) . '/include/ai_chat.php';
|
||||
the two red ones do not. */
|
||||
.vv-au-stat-n.ok2 { color:#4a7a4a; }
|
||||
|
||||
/* Fixed columns for the proxy table. With auto layout each figure column sized itself to its own
|
||||
widest cell, so the three of them were three different widths and every column shifted whenever
|
||||
the numbers changed. Declared once here and the host column takes whatever is left. */
|
||||
.vv-au-tbl.fixed { table-layout:fixed;min-width:960px; }
|
||||
.vv-au-tblwrap { overflow-x:auto; }
|
||||
.vv-au-c-flags { width:154px; }
|
||||
/* 60 spark bars at 2px plus 1px gaps is 179px, and the cell has 10px padding either side. Every
|
||||
figure column gets that same width whether or not it draws bars. */
|
||||
.vv-au-c-stat { width:200px; }
|
||||
.vv-au-c-tog { width:56px; }
|
||||
.vv-au-c-act { width:64px; }
|
||||
|
||||
/* ── Why ─────────────────────────────────────────────────────────────────── */
|
||||
/* On the figure's own line, not under the strip: the strip is 179px of fixed-width bars and is what
|
||||
sets this column's width, so anything below it would widen every row for the four that need it. */
|
||||
.vv-au-upl { display:flex;align-items:center;justify-content:flex-end;gap:6px; }
|
||||
.vv-au-why { font-size:9px;padding:1px 5px;border-radius:2px;background:#1f1200;
|
||||
border:1px solid #3a2000;color:#ff9800;cursor:pointer;line-height:1.4; }
|
||||
.vv-au-why:hover{ background:#2a1900;color:#ffb74d; }
|
||||
.vv-au-why:disabled { opacity:.4;cursor:default; }
|
||||
.vv-au-why-wait { font-size:11px;color:#666;padding:14px 2px;line-height:1.5; }
|
||||
/* The findings are the answer, so they get the top of the dialog and enough weight to be read as
|
||||
sentences rather than as another row of status. */
|
||||
.vv-au-find { font-size:11px;line-height:1.5;padding:8px 10px;border-radius:3px;margin-bottom:6px;
|
||||
border-left:2px solid #333;background:#141414;color:#888; }
|
||||
.vv-au-find.bad { border-left-color:#ef5350;color:#d88; }
|
||||
.vv-au-find.warn{ border-left-color:#ff9800;color:#b98a4a; }
|
||||
.vv-au-find.info{ border-left-color:#2a4a6a;color:#777; }
|
||||
.vv-au-why-grid { display:grid;grid-template-columns:auto 1fr;gap:3px 12px;margin:12px 0 4px;font-size:11px; }
|
||||
.vv-au-why-k { color:#444;white-space:nowrap; }
|
||||
.vv-au-why-v { color:#999;overflow-wrap:anywhere; }
|
||||
.vv-au-why-sec { font-size:10px;color:#444;text-transform:uppercase;letter-spacing:.06em;
|
||||
margin:14px 0 6px;border-top:1px solid #222;padding-top:10px; }
|
||||
.vv-au-why-h { margin-bottom:10px; }
|
||||
.vv-au-why-hd { font-size:11px;color:#999;display:flex;justify-content:space-between;gap:10px; }
|
||||
.vv-au-why-bad { font-size:10px;color:#8a6a2a;margin:2px 0 3px; }
|
||||
.vv-au-why-ev { font-size:10px;color:#666;margin-top:2px; }
|
||||
.vv-au-why-note { font-size:10px;color:#4a4a4a;margin-top:2px;line-height:1.45; }
|
||||
|
||||
/* ── Access simulator ────────────────────────────────────────────────────── */
|
||||
.vv-au-sim { padding:12px 14px;margin-bottom:10px; }
|
||||
.vv-au-sim-row { display:flex;align-items:center;gap:8px;flex-wrap:wrap; }
|
||||
.vv-au-sim-row .vv-au-label { margin-bottom:0; }
|
||||
.vv-au-sim-row .vv-au-select { width:auto;min-width:150px;flex:0 1 auto; }
|
||||
.vv-au-sim-path { width:110px;flex:0 0 auto; }
|
||||
/* One word, because that is the question. Sized to be readable from the second screen. */
|
||||
.vv-au-sim-verdict { font-size:20px;font-weight:700;letter-spacing:.02em;margin:14px 0 10px; }
|
||||
.vv-au-sim-verdict.ok { color:#4a7a4a; }
|
||||
.vv-au-sim-verdict.warn { color:#ff9800; }
|
||||
.vv-au-sim-verdict.bad { color:#ef5350; }
|
||||
.vv-au-sim-t { display:flex;gap:8px;font-size:10px;color:#555;padding:3px 0;align-items:baseline; }
|
||||
.vv-au-sim-t.hit{ color:#999; }
|
||||
.vv-au-sim-n { color:#333;min-width:14px; }
|
||||
.vv-au-sim-t.hit .vv-au-sim-n { color:#4a7a4a; }
|
||||
.vv-au-sim-l { min-width:130px;color:#666; }
|
||||
.vv-au-sim-t.hit .vv-au-sim-l { color:#bbb;font-weight:600; }
|
||||
.vv-au-sim-w { color:#444;overflow-wrap:anywhere; }
|
||||
.vv-au-sim-t.hit .vv-au-sim-w { color:#777; }
|
||||
|
||||
/* ── Renewal triage ──────────────────────────────────────────────────────── */
|
||||
/* Root causes carry the marker; consequences are dimmed. Sorting these by count would put rate
|
||||
limiting first every time, and rate limiting is the one thing here never worth fixing directly. */
|
||||
.vv-au-tri-row { display:flex;gap:10px;padding:6px 0;border-bottom:1px solid #1a1a1a;align-items:baseline; }
|
||||
.vv-au-tri-row:last-of-type { border-bottom:none; }
|
||||
.vv-au-tri-row.root .vv-au-tri-n { color:#ff9800; }
|
||||
.vv-au-tri-n { font-size:15px;font-weight:700;color:#555;min-width:34px;text-align:right; }
|
||||
.vv-au-tri-w { font-size:11px;color:#888;line-height:1.5; }
|
||||
.vv-au-tri-d { font-size:10px;color:#4a4a4a;font-family:monospace;margin-top:2px;overflow-wrap:anywhere; }
|
||||
.vv-au-ok { color:#4a7a4a; }
|
||||
.vv-au-bad { color:#ef5350; }
|
||||
.vv-au-warn { color:#ff9800; }
|
||||
.vv-au-dim { color:#444; }
|
||||
|
||||
/* One bar per probe, last hour. The shape is the part a percentage throws away: one long outage
|
||||
and sixty scattered blips are the same 50% and completely different problems. */
|
||||
.vv-au-spark { display:flex;gap:1px;justify-content:flex-end;align-items:flex-end;height:11px;margin-top:3px; }
|
||||
@@ -361,12 +436,18 @@ $tabs = ['proxies' => 'Proxies', 'users' => 'Users & Groups',
|
||||
</div>
|
||||
<div class="vv-au-card">
|
||||
<div class="vv-au-loading" id="vv-au-proxy-loading">Loading…</div>
|
||||
<table class="vv-au-tbl" id="vv-au-proxy-tbl" style="display:none">
|
||||
<div class="vv-au-tblwrap" id="vv-au-proxy-wrap" style="display:none">
|
||||
<table class="vv-au-tbl fixed" id="vv-au-proxy-tbl">
|
||||
<colgroup>
|
||||
<col><col class="vv-au-c-flags"><col class="vv-au-c-stat"><col class="vv-au-c-stat">
|
||||
<col class="vv-au-c-stat"><col class="vv-au-c-tog"><col class="vv-au-c-act">
|
||||
</colgroup>
|
||||
<thead><tr>
|
||||
<th>Host</th><th>Flags</th><th style="text-align:right">Uptime</th><th style="text-align:right">Requests</th><th style="text-align:right">Errors</th><th>On</th><th></th>
|
||||
</tr></thead>
|
||||
<tbody id="vv-au-proxy-body"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="vv-au-empty" id="vv-au-proxy-empty" style="display:none">No proxy hosts configured.</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -418,6 +499,22 @@ $tabs = ['proxies' => 'Proxies', 'users' => 'Users & Groups',
|
||||
<span class="vv-au-dirty-note" id="vv-au-ac-dirty" style="margin-left:auto"></span>
|
||||
<button class="vv-au-btn green" id="vv-au-ac-save">Save & Restart Authelia</button>
|
||||
</div>
|
||||
<!-- Whether one person can open one URL is decided by three objects on three different tabs: the
|
||||
NPM host (is it handed to Authelia at all), the rule list below (which rule wins, in file
|
||||
order), and the LDAP group the rule names. This asks the question directly instead of making
|
||||
someone hold all three in their head — and it reads the Authelia instance the chosen host
|
||||
actually talks to, which is not always the one this tab edits. -->
|
||||
<div class="vv-au-card vv-au-sim">
|
||||
<div class="vv-au-sim-row">
|
||||
<span class="vv-au-label">Can</span>
|
||||
<select class="vv-au-select" id="vv-au-sim-user"></select>
|
||||
<span class="vv-au-label">open</span>
|
||||
<select class="vv-au-select" id="vv-au-sim-dom"></select>
|
||||
<input class="vv-au-input vv-au-sim-path" id="vv-au-sim-path" value="/" title="Path — rules carrying a resources pattern only apply to the paths they name">
|
||||
<button class="vv-au-btn prim" id="vv-au-sim-run">Test</button>
|
||||
</div>
|
||||
<div id="vv-au-sim-out"></div>
|
||||
</div>
|
||||
<div class="vv-au-sec-bar">
|
||||
<span class="vv-au-sec-title" id="vv-au-acl-count"></span>
|
||||
<button class="vv-au-btn prim" id="vv-au-rule-add">+ Add Rule</button>
|
||||
@@ -437,8 +534,13 @@ $tabs = ['proxies' => 'Proxies', 'users' => 'Users & Groups',
|
||||
<div class="vv-au-panel<?= $first==="certs" ? " active" : "" ?>" id="vv-au-panel-certs">
|
||||
<div class="vv-au-sec-bar">
|
||||
<span class="vv-au-sec-title" id="vv-au-cert-ts"></span>
|
||||
<!-- The counts on this tab say a domain stopped renewing. certbot's logs say why, and they are
|
||||
639 MB of Python tracebacks — which is why the last answer to that question was somebody
|
||||
reading them by hand. This reads the newest runs and names the categories. -->
|
||||
<button class="vv-au-btn" id="vv-au-cert-triage">Why renewals fail</button>
|
||||
<button class="vv-au-btn prim" id="vv-au-cert-run">Refresh</button>
|
||||
</div>
|
||||
<div id="vv-au-cert-triage-out"></div>
|
||||
<div class="vv-au-cert-grid" id="vv-au-cert-grid">
|
||||
<div class="vv-au-loading">Loading…</div>
|
||||
</div>
|
||||
@@ -536,8 +638,11 @@ function _esc(s) {
|
||||
return String(s||'').replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"');
|
||||
}
|
||||
|
||||
function _get(action, cb) {
|
||||
fetch(API + '?action=' + action)
|
||||
// params is optional and encoded rather than concatenated — the only caller passing one passes an
|
||||
// id, but a query string built by hand is how a value with an & in it silently becomes two.
|
||||
function _get(action, cb, params) {
|
||||
const qs = params ? '&' + new URLSearchParams(params).toString() : '';
|
||||
fetch(API + '?action=' + action + qs)
|
||||
.then(r => r.json()).then(cb)
|
||||
.catch(e => cb({ ok: false, error: String(e) }));
|
||||
}
|
||||
@@ -639,7 +744,7 @@ function _loadTab(tab) {
|
||||
// ── Proxies ───────────────────────────────────────────────────────────────────
|
||||
function _loadProxies() {
|
||||
const loading = document.getElementById('vv-au-proxy-loading');
|
||||
const tbl = document.getElementById('vv-au-proxy-tbl');
|
||||
const tbl = document.getElementById('vv-au-proxy-wrap');
|
||||
const empty = document.getElementById('vv-au-proxy-empty');
|
||||
loading.style.display = 'block';
|
||||
tbl.style.display = 'none';
|
||||
@@ -651,7 +756,7 @@ function _loadProxies() {
|
||||
if (!certsLoaded || !proxiesLoaded) return;
|
||||
loading.style.display = 'none';
|
||||
if (!_proxies.length) { empty.style.display = 'block'; return; }
|
||||
tbl.style.display = 'table';
|
||||
tbl.style.display = 'block';
|
||||
_renderProxies();
|
||||
}
|
||||
|
||||
@@ -765,10 +870,189 @@ function _upCell(p) {
|
||||
? 'DOWN — ' + (worst.last_detail || 'no response')
|
||||
: 'up' + (worst.last_ms ? ' · ' + worst.last_ms + 'ms' : '');
|
||||
|
||||
return `<div class="vv-au-stat-n${cls}" title="${_esc(title)}">${pct === null ? '—' : (pct >= 99.95 ? '100' : pct.toFixed(pct >= 99 ? 2 : 1))}<span class="vv-au-stat-u">% 24h</span></div>
|
||||
// Offered only where there is something to explain. A why? on all thirty-five rows is a button
|
||||
// nobody reads; on the four that are not fine it is the next thing you were going to do anyway.
|
||||
//
|
||||
// 96 rather than 100 because a probe every minute makes 99.9% an ordinary week — one missed
|
||||
// sample in a day is 99.93 and means nothing. Below 96 is roughly an hour lost in a day, which is
|
||||
// always something. Anything currently down qualifies whatever its percentage says.
|
||||
const needWhy = down || (pct !== null && pct < 96);
|
||||
const why = needWhy
|
||||
? `<button class="vv-au-why" data-why="${p.id}" title="Go and look at what is behind this figure">why?</button>`
|
||||
: '';
|
||||
|
||||
return `<div class="vv-au-upl">${why}<div class="vv-au-stat-n${cls}" title="${_esc(title)}">${pct === null ? '—' : (pct >= 99.95 ? '100' : pct.toFixed(pct >= 99 ? 2 : 1))}<span class="vv-au-stat-u">% 24h</span></div></div>
|
||||
<div class="vv-au-spark" title="last hour, one bar per minute">${bars}</div>`;
|
||||
}
|
||||
|
||||
// ── Why ───────────────────────────────────────────────────────────────────────
|
||||
// Held so the assistant hand-off has something to send without asking the server twice. Cleared
|
||||
// with the dialog, because a brief describing a probe from ten minutes ago is worse than no brief.
|
||||
let _why = null;
|
||||
|
||||
function _whyModal(id, btn) {
|
||||
const p = _proxies.find(x => x.id === id);
|
||||
const name = (p?.domain_names || []).join(', ') || ('host ' + id);
|
||||
_why = null;
|
||||
// Disabled for the duration. The check takes seconds and opens sockets, and a second press while
|
||||
// the first is in flight probes the host twice to answer the same question.
|
||||
if (btn) btn.disabled = true;
|
||||
|
||||
// Named while it works, because it genuinely takes a few seconds and a silent dialog reads as a
|
||||
// hang. What it is about to do is listed rather than hidden behind a spinner — this opens
|
||||
// connections, and an operator watching a limping host should know the page is about to touch it.
|
||||
_modal(`<h3>Why — ${_esc(name)}</h3>
|
||||
<div class="vv-au-why-wait">Looking… connecting to the service behind this host, asking the
|
||||
domain itself, and reading the probe history.</div>`, true);
|
||||
|
||||
_get('npm_why', r => {
|
||||
if (btn) btn.disabled = false;
|
||||
if (!r || !r.ok) {
|
||||
_modal(`<h3>Why — ${_esc(name)}</h3>
|
||||
<div class="vv-au-err show">${_esc(r?.error || 'The check failed')}</div>
|
||||
<div class="vv-au-modal-acts"><button class="vv-au-btn" id="wm-close">Close</button></div>`, true);
|
||||
document.getElementById('wm-close').onclick = _closeModal;
|
||||
return;
|
||||
}
|
||||
_why = r;
|
||||
_modal(_whyHtml(r, name), true);
|
||||
document.getElementById('wm-close').onclick = _closeModal;
|
||||
const ask = document.getElementById('wm-ask');
|
||||
if (ask) ask.onclick = () => _whyAsk(name);
|
||||
}, { id });
|
||||
}
|
||||
|
||||
// Hand the measurements to the assistant. The dialog closes first: the chat card is at the foot of
|
||||
// the page and an answer streaming in behind an overlay is an answer nobody sees.
|
||||
function _whyAsk(name) {
|
||||
const chat = window.__vvAiChat && window.__vvAiChat['vv-au-ai'];
|
||||
if (!chat || !_why) return;
|
||||
if (chat.busy()) { vvAlert('The assistant is still answering the previous question.'); return; }
|
||||
const brief = _whyBrief(_why, name);
|
||||
_closeModal();
|
||||
// retarget, so the thread is about this host and not appended to whatever was asked before it.
|
||||
// Same reasoning as the Watchdog tab: a troubleshooting thread about one host must not bleed into
|
||||
// a question about another.
|
||||
chat.retarget('troubleshoot', name, 'now looking at ' + name);
|
||||
const input = document.getElementById('vv-au-ai-input');
|
||||
if (input) input.value = brief;
|
||||
chat.send();
|
||||
const card = document.getElementById('vv-au-ai-card');
|
||||
if (card) card.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
|
||||
}
|
||||
|
||||
function _whyHtml(r, name) {
|
||||
const lvl = { bad: 'bad', warn: 'warn', info: 'info' };
|
||||
|
||||
// The findings first and everything else under them. The evidence is what makes the answer
|
||||
// checkable, but it is not the answer, and a dialog that opens on a table of status codes makes
|
||||
// the reader do the work the check just did.
|
||||
const findings = (r.findings || []).map(f =>
|
||||
`<div class="vv-au-find ${lvl[f.level] || 'info'}">${_esc(f.text)}</div>`).join('');
|
||||
|
||||
const up = r.upstream || {};
|
||||
const tcp = up.tcp || {};
|
||||
const rows = [];
|
||||
|
||||
rows.push(['Forwards to', _esc(r.forward)
|
||||
+ (up.container ? ` <span class="vv-au-m${up.container.running ? '' : ' auth'}">${_esc(up.container.name)} · ${_esc(up.container.status)}</span>` : '')]);
|
||||
// The address actually knocked on, not the one written in NPM. They differ whenever the forward
|
||||
// host is a docker name, and a result attributed to the wrong address is worse than no result.
|
||||
rows.push([tcp.ok ? 'Port open' : 'Port', (tcp.ok
|
||||
? `<span class="vv-au-ok">${_esc(up.probed || '')}:${up.port}</span> <span class="vv-au-dim">${tcp.ms}ms</span>`
|
||||
: `<span class="vv-au-bad">${_esc(up.probed || '')}:${up.port} — ${_esc(tcp.err || 'no answer')}</span>`)
|
||||
+ (up.note ? `<div class="vv-au-why-note">${_esc(up.note)}</div>` : '')]);
|
||||
if (up.http)
|
||||
rows.push(['Service answers', up.http.code
|
||||
? `<span class="${up.http.code >= 500 ? 'vv-au-bad' : 'vv-au-ok'}">HTTP ${up.http.code}</span> <span class="vv-au-dim">${up.http.ms}ms</span>`
|
||||
: `<span class="vv-au-bad">${_esc(up.http.err || 'no response')}</span>`]);
|
||||
|
||||
// Both halves of the certificate: what NPM records and what the handshake did. They disagree more
|
||||
// often than they should — a cert with a month left still fails if the chain being served is wrong.
|
||||
if (r.cert) {
|
||||
const d = r.cert.days_left;
|
||||
rows.push(['Certificate', `${_esc(r.cert.name)} <span class="${d !== null && d < 0 ? 'vv-au-bad' : (d !== null && d <= 14 ? 'vv-au-warn' : 'vv-au-dim')}">`
|
||||
+ (d === null ? 'no expiry recorded' : (d < 0 ? `expired ${Math.abs(d)}d ago` : `${d}d left`)) + '</span>']);
|
||||
}
|
||||
if (r.guarded) rows.push(['Protection', '<span class="vv-au-dim">behind Authelia (auth_request)</span>']);
|
||||
if (!r.enabled) rows.push(['State', '<span class="vv-au-bad">disabled in NPM</span>']);
|
||||
|
||||
for (const [d, l] of Object.entries(r.live || {})) {
|
||||
const okNow = l.code > 0 && l.code < 500;
|
||||
rows.push([_esc(d), (okNow ? `<span class="vv-au-ok">HTTP ${l.code}</span> <span class="vv-au-dim">${l.ms}ms</span>`
|
||||
: `<span class="vv-au-bad">${l.code ? 'HTTP ' + l.code : _esc(l.err || 'no answer')}</span>`)
|
||||
+ (l.insecure ? ` <span class="vv-au-warn">— answers ${l.insecure.code} without certificate verification</span>` : '')]);
|
||||
}
|
||||
|
||||
const t = r.traffic;
|
||||
if (t) rows.push(['Logged traffic', `${_fmtNum(t.requests || 0)} requests · ${_fmtNum(t.s4xx || 0)} 4xx · ${_fmtNum(t.s5xx || 0)} 5xx`
|
||||
+ (t.last_seen ? ` <span class="vv-au-dim">· last ${_ago(t.last_seen)} ago</span>` : ' <span class="vv-au-dim">· never hit</span>')]);
|
||||
|
||||
const grid = rows.map(([k, v]) =>
|
||||
`<div class="vv-au-why-k">${k}</div><div class="vv-au-why-v">${v}</div>`).join('');
|
||||
|
||||
// The recorded history, per domain. Kept last and kept short: it is the part that says whether
|
||||
// this is happening now or already over, which only matters once you know what "this" is.
|
||||
const hist = Object.entries(r.history || {}).map(([d, h]) => {
|
||||
const ev = (h.events || []).map(e =>
|
||||
`<div class="vv-au-why-ev"><span class="${e.to === 'up' ? 'vv-au-ok' : 'vv-au-bad'}">${_esc(String(e.to).toUpperCase())}</span>
|
||||
<span class="vv-au-dim">${_ago(e.ts)} ago</span> ${_esc(e.detail || '')}</div>`).join('');
|
||||
const bad = (h.bad_hours || []).map(b =>
|
||||
`${String(b.hour).slice(8, 10)}:00 <span class="vv-au-dim">${b.up}/${b.total}</span>`).join(' · ');
|
||||
return `<div class="vv-au-why-h">
|
||||
<div class="vv-au-why-hd">${_esc(d)}
|
||||
<span class="vv-au-dim">${h.h1 === null ? '—' : h.h1 + '% 1h'} · ${h.h24 === null ? '—' : h.h24 + '% 24h'} · ${h.d30 === null ? '—' : h.d30 + '% 30d'}</span></div>
|
||||
${bad ? `<div class="vv-au-why-bad">Hours that lost samples: ${bad}</div>` : ''}
|
||||
${ev || '<div class="vv-au-why-ev vv-au-dim">no state changes recorded</div>'}
|
||||
</div>`;
|
||||
}).join('');
|
||||
|
||||
// Offered only where there is a model to ask. The findings above are the page's own reading and
|
||||
// stand without it — the assistant is for the step after, which is what to do about it.
|
||||
const ask = document.getElementById('vv-au-ai-chat')
|
||||
? '<button class="vv-au-btn prim" id="wm-ask">Ask the assistant</button>' : '';
|
||||
|
||||
return `<h3>Why — ${_esc(name)}</h3>
|
||||
${findings}
|
||||
<div class="vv-au-why-grid">${grid}</div>
|
||||
${hist ? `<div class="vv-au-why-sec">Recorded history</div>${hist}` : ''}
|
||||
<div class="vv-au-modal-acts">${ask}<button class="vv-au-btn" id="wm-close">Close</button></div>`;
|
||||
}
|
||||
|
||||
// The brief the assistant is given. Deliberately the same facts the dialog just showed, written out
|
||||
// rather than summarised: a model asked "why is this host down" with no evidence answers from the
|
||||
// general shape of the question, and the whole point of the check above is that it went and looked.
|
||||
function _whyBrief(r, name) {
|
||||
const L = [];
|
||||
L.push(`Proxy host ${name} on this machine is not at full uptime. Here is what the Auth tab just measured.`);
|
||||
L.push(`Forwards to ${r.forward}${r.guarded ? ', behind Authelia via auth_request' : ''}${r.enabled ? '' : '. The host is DISABLED in NPM'}.`);
|
||||
|
||||
const up = r.upstream || {}, tcp = up.tcp || {};
|
||||
if (up.note) L.push(`Note on the check itself: ${up.note}`);
|
||||
L.push(`TCP to ${up.probed}:${up.port} — ${tcp.ok ? 'open in ' + tcp.ms + 'ms' : 'failed: ' + (tcp.err || 'no reason')}.`);
|
||||
if (up.container) L.push(`That target is container ${up.container.name}, currently ${up.container.status}.`);
|
||||
if (up.http) L.push(`Asked the service directly: ${up.http.code ? 'HTTP ' + up.http.code : 'no response (' + (up.http.err || '') + ')'}.`);
|
||||
if (r.cert) L.push(`Certificate ${r.cert.name}: ${r.cert.days_left === null ? 'no expiry recorded' : r.cert.days_left + ' days left'}.`);
|
||||
|
||||
for (const [d, l] of Object.entries(r.live || {}))
|
||||
L.push(`Through the proxy, https://${d}/ — ${l.code ? 'HTTP ' + l.code + ' in ' + l.ms + 'ms' : 'no answer: ' + (l.err || '')}`
|
||||
+ (l.insecure ? `; with certificate verification off it answers ${l.insecure.code}` : '') + '.');
|
||||
|
||||
const t = r.traffic;
|
||||
if (t) L.push(`Access log totals: ${t.requests || 0} requests, ${t.s4xx || 0} 4xx, ${t.s5xx || 0} 5xx.`);
|
||||
|
||||
for (const [d, h] of Object.entries(r.history || {})) {
|
||||
L.push(`${d}: ${h.h1 === null ? '?' : h.h1}% in the last hour, ${h.h24 === null ? '?' : h.h24}% over 24h, ${h.d30 === null ? '?' : h.d30}% over 30 days; currently ${h.state || 'unknown'}, last probe said "${h.last_detail || '?'}".`);
|
||||
for (const e of (h.events || []).slice(0, 5))
|
||||
L.push(` went ${e.to} ${_ago(e.ts)} ago — ${e.detail || ''}`);
|
||||
}
|
||||
|
||||
L.push('');
|
||||
L.push('What is the most likely cause, and what should I check or change first? '
|
||||
+ 'If the evidence above already settles it, say so rather than listing possibilities.');
|
||||
return L.join('\n');
|
||||
}
|
||||
|
||||
function _certName(id) {
|
||||
const c = _certs.find(x => String(x.id) === String(id));
|
||||
return c ? (c.nice_name || (c.domain_names || []).join(', ')) : 'certificate ' + id;
|
||||
@@ -964,6 +1248,9 @@ _on('vv-au-panel-proxies', 'click', async e => {
|
||||
});
|
||||
return;
|
||||
}
|
||||
// Why
|
||||
const whyBtn = e.target.closest('[data-why]');
|
||||
if (whyBtn) { _whyModal(parseInt(whyBtn.dataset.why), whyBtn); return; }
|
||||
// Edit
|
||||
const editBtn = e.target.closest('[data-proxy-edit]');
|
||||
if (editBtn) { _proxyModal(parseInt(editBtn.dataset.proxyEdit)); return; }
|
||||
@@ -1607,8 +1894,129 @@ _on('vv-au-panel-users', 'click', async e => {
|
||||
}
|
||||
});
|
||||
|
||||
// ── Access simulator ──────────────────────────────────────────────────────────
|
||||
// The dropdowns come from the other two panels' data, which this panel does not otherwise load —
|
||||
// AUTH_STACK decides which panels exist and _loadTab only fetches what the open one needs, so both
|
||||
// lists are fetched here rather than assumed to be sitting in the page already.
|
||||
function _simFill() {
|
||||
const dom = document.getElementById('vv-au-sim-dom');
|
||||
const user = document.getElementById('vv-au-sim-user');
|
||||
if (!dom || !user) return;
|
||||
|
||||
const fillDom = () => {
|
||||
// Wildcards are dropped: they are not a hostname anyone opens, so simulating one answers a
|
||||
// question nobody can ask. Disabled hosts stay in — "why does this not work" is often "it is
|
||||
// switched off", and hiding them hides the answer.
|
||||
const seen = new Set();
|
||||
_proxies.forEach(p => (p.domain_names || []).forEach(d => { if (!String(d).includes('*')) seen.add(String(d).toLowerCase()); }));
|
||||
dom.innerHTML = [...seen].sort().map(d => `<option value="${_esc(d)}">${_esc(d)}</option>`).join('')
|
||||
|| '<option value="">no proxy hosts</option>';
|
||||
};
|
||||
const fillUser = () => {
|
||||
user.innerHTML = _users.slice().sort((a, b) => String(a.id).localeCompare(String(b.id)))
|
||||
.map(u => `<option value="${_esc(u.id)}">${_esc(u.id)}</option>`).join('')
|
||||
|| '<option value="">no users</option>';
|
||||
};
|
||||
|
||||
if (_proxies.length) fillDom();
|
||||
else _get('npm_proxies', r => { if (r.ok) { _proxies = r.proxies || []; fillDom(); } });
|
||||
if (_users.length) fillUser();
|
||||
else _get('lldap_users', r => { if (r.ok) { _users = r.users || []; fillUser(); } });
|
||||
}
|
||||
|
||||
// Held for the assistant hand-off, same as the proxy check.
|
||||
let _sim = null;
|
||||
|
||||
function _simRun() {
|
||||
const out = document.getElementById('vv-au-sim-out');
|
||||
const btn = document.getElementById('vv-au-sim-run');
|
||||
const dom = document.getElementById('vv-au-sim-dom').value;
|
||||
const uid = document.getElementById('vv-au-sim-user').value;
|
||||
const path = document.getElementById('vv-au-sim-path').value || '/';
|
||||
if (!dom) return;
|
||||
|
||||
_sim = null;
|
||||
btn.disabled = true;
|
||||
out.innerHTML = '<div class="vv-au-why-wait">Reading the proxy host, the rules of the Authelia it talks to, and the directory…</div>';
|
||||
|
||||
_get('access_check', r => {
|
||||
btn.disabled = false;
|
||||
if (!r || !r.ok) { out.innerHTML = `<div class="vv-au-err show">${_esc(r?.error || 'The check failed')}</div>`; return; }
|
||||
_sim = r;
|
||||
out.innerHTML = _simHtml(r);
|
||||
const ask = document.getElementById('vv-au-sim-ask');
|
||||
if (ask) ask.onclick = () => _simAsk(r);
|
||||
}, { domain: dom, uid, path });
|
||||
}
|
||||
|
||||
function _simHtml(r) {
|
||||
// The verdict as one word, because that is the question. Bypass is coloured as a warning rather
|
||||
// than a success — reaching a page without being asked to log in is only good news if it was
|
||||
// meant to be public, and the findings below say which case this is.
|
||||
const pol = r.policy || 'unknown';
|
||||
const cls = pol === 'deny' ? 'bad' : (pol === 'bypass' ? 'warn' : 'ok');
|
||||
const verdict = `<div class="vv-au-sim-verdict ${cls}">${_esc(pol)}</div>`;
|
||||
|
||||
const findings = (r.findings || []).map(f =>
|
||||
`<div class="vv-au-find ${_esc(f.level || 'info')}">${_esc(f.text)}</div>`).join('');
|
||||
|
||||
// Every rule, including the ones that did nothing. Which rule you expected to win and why it was
|
||||
// stepped over is the part that is impossible to see from the cards below.
|
||||
const trace = (r.trace || []).map(t =>
|
||||
`<div class="vv-au-sim-t${t.applied ? ' hit' : ''}">
|
||||
<span class="vv-au-sim-n">${t.n}</span>
|
||||
<span class="vv-au-sim-l">${_esc(t.label)}</span>
|
||||
<span class="vv-au-sim-w">${_esc(t.why)}${t.applied ? ' → ' + _esc(t.policy || '') : ''}</span>
|
||||
</div>`).join('');
|
||||
|
||||
const groups = (r.groups || []).length
|
||||
? (r.groups || []).map(g => `<span class="vv-au-m">${_esc(g)}</span>`).join(' ')
|
||||
: '<span class="vv-au-dim">no groups</span>';
|
||||
|
||||
const ask = document.getElementById('vv-au-ai-chat')
|
||||
? '<button class="vv-au-btn prim" id="vv-au-sim-ask">Ask the assistant</button>' : '';
|
||||
|
||||
return `${verdict}${findings}
|
||||
<div class="vv-au-why-grid">
|
||||
<div class="vv-au-why-k">Groups</div><div class="vv-au-why-v">${groups}</div>
|
||||
${r.forward ? `<div class="vv-au-why-k">Forwards to</div><div class="vv-au-why-v">${_esc(r.forward)}</div>` : ''}
|
||||
${r.authelia && r.authelia.container ? `<div class="vv-au-why-k">Decided by</div><div class="vv-au-why-v">${_esc(r.authelia.container)}<div class="vv-au-why-note">${_esc(r.authelia.config || '')}</div></div>` : ''}
|
||||
${r.default_policy ? `<div class="vv-au-why-k">Default policy</div><div class="vv-au-why-v">${_esc(r.default_policy)}</div>` : ''}
|
||||
</div>
|
||||
${trace ? `<div class="vv-au-why-sec">Every rule, in file order</div>${trace}` : ''}
|
||||
${ask ? `<div class="vv-au-modal-acts">${ask}</div>` : ''}`;
|
||||
}
|
||||
|
||||
function _simAsk(r) {
|
||||
const chat = window.__vvAiChat && window.__vvAiChat['vv-au-ai'];
|
||||
if (!chat) return;
|
||||
if (chat.busy()) { vvAlert('The assistant is still answering the previous question.'); return; }
|
||||
|
||||
const L = [];
|
||||
L.push(`On this machine, can ${r.uid || 'anyone'} open https://${r.domain}${r.path}? Here is what the Auth tab worked out.`);
|
||||
L.push(`The proxy host ${r.served ? 'exists' : 'does not exist'}${r.served && !r.enabled ? ' but is disabled' : ''}${r.forward ? ', forwarding to ' + r.forward : ''}.`);
|
||||
L.push(`${r.uid || 'The user'} is in these LDAP groups: ${(r.groups || []).join(', ') || 'none'}.`);
|
||||
if (r.authelia && r.authelia.container)
|
||||
L.push(`Authentication is decided by ${r.authelia.container}, config ${r.authelia.config || 'not found'}${r.authelia.is_configured ? '' : ' — which is not the instance this tab edits'}.`);
|
||||
L.push(`Rules were walked in file order:`);
|
||||
for (const t of (r.trace || []))
|
||||
L.push(` ${t.n}. ${t.label} — ${t.applied ? 'APPLIED, policy ' + (t.policy || '') : 'skipped: ' + t.why}`);
|
||||
L.push(`Default policy is ${r.default_policy || 'unknown'}. The resulting policy is ${r.policy || 'unknown'}.`);
|
||||
L.push('');
|
||||
L.push('Is that the intended outcome? If not, say which rule to change and how — be specific about '
|
||||
+ 'rule order, because Authelia stops at the first rule that matches on every axis.');
|
||||
|
||||
chat.retarget('troubleshoot', r.domain, 'now looking at access to ' + r.domain);
|
||||
const input = document.getElementById('vv-au-ai-input');
|
||||
if (input) input.value = L.join('\n');
|
||||
chat.send();
|
||||
const card = document.getElementById('vv-au-ai-card');
|
||||
if (card) card.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
|
||||
}
|
||||
|
||||
// ── Access Control ────────────────────────────────────────────────────────────
|
||||
function _loadAcl() {
|
||||
_simFill();
|
||||
const loading = document.getElementById('vv-au-acl-loading');
|
||||
const grid = document.getElementById('vv-au-acl-body');
|
||||
const empty = document.getElementById('vv-au-acl-empty');
|
||||
@@ -1842,6 +2250,10 @@ _on('vv-au-panel-acl', 'input', e => {
|
||||
// Enter adds a row underneath and moves to it, so a list of fourteen can be typed straight
|
||||
// through instead of returning to the add button between each one.
|
||||
_on('vv-au-panel-acl', 'keydown', e => {
|
||||
// Enter in the path box runs the simulation. Checked before the domain-input handler below,
|
||||
// which would otherwise never see it anyway but reads as if it might.
|
||||
if (e.target.id === 'vv-au-sim-path' && e.key === 'Enter') { e.preventDefault(); _simRun(); return; }
|
||||
|
||||
const inp = e.target.closest('.vv-au-dominput');
|
||||
if (!inp || e.key !== 'Enter') return;
|
||||
e.preventDefault();
|
||||
@@ -1860,6 +2272,7 @@ function _focusDomain(r, d) {
|
||||
|
||||
// ACL event delegation
|
||||
_on('vv-au-panel-acl', 'click', async e => {
|
||||
if (e.target.id === 'vv-au-sim-run') { _simRun(); return; }
|
||||
if (e.target.id === 'vv-au-rule-add') { _ruleModal(null); return; }
|
||||
|
||||
const domAdd = e.target.closest('[data-dom-add]');
|
||||
@@ -2112,6 +2525,75 @@ _on('vv-au-cert-run', 'click', function() {
|
||||
_loadCerts(() => { btn.disabled = false; btn.textContent = 'Refresh'; });
|
||||
});
|
||||
|
||||
// ── Renewal triage ────────────────────────────────────────────────────────────
|
||||
let _triage = null;
|
||||
|
||||
_on('vv-au-cert-triage', 'click', function() {
|
||||
const btn = this, out = document.getElementById('vv-au-cert-triage-out');
|
||||
btn.disabled = true;
|
||||
out.innerHTML = '<div class="vv-au-why-wait">Reading certbot\'s logs…</div>';
|
||||
|
||||
fetch(CERT_API + '?action=triage').then(r => r.json()).then(r => {
|
||||
btn.disabled = false;
|
||||
if (!r || !r.ok) { out.innerHTML = `<div class="vv-au-err show">${_esc(r?.error || 'Triage failed')}</div>`; return; }
|
||||
_triage = r;
|
||||
out.innerHTML = _triageHtml(r);
|
||||
const ask = document.getElementById('vv-au-triage-ask');
|
||||
if (ask) ask.onclick = () => _triageAsk(r);
|
||||
}).catch(e => { btn.disabled = false; out.innerHTML = `<div class="vv-au-err show">${_esc(String(e))}</div>`; });
|
||||
});
|
||||
|
||||
function _triageHtml(r) {
|
||||
if (!r.total)
|
||||
return `<div class="vv-au-card vv-au-sim"><div class="vv-au-find info">No renewal failures in the
|
||||
last ${r.files_read} certbot runs. Nothing is failing to renew right now.</div></div>`;
|
||||
|
||||
// Causes first and consequences after, marked as such. Sorted by count would put rate limiting
|
||||
// at the top of every one of these, and rate limiting is almost never the thing to fix.
|
||||
const cats = (r.categories || []).map(c => `
|
||||
<div class="vv-au-tri-row${c.root ? ' root' : ''}">
|
||||
<span class="vv-au-tri-n">${c.count}</span>
|
||||
<span class="vv-au-tri-w">${_esc(c.what)}${c.root ? '' : ' <span class="vv-au-dim">— a consequence, not a cause</span>'}
|
||||
${(c.domains || []).length ? `<div class="vv-au-tri-d">${(c.domains || []).map(d => _esc(d)).join(' · ')}</div>` : ''}
|
||||
</span>
|
||||
</div>`).join('');
|
||||
|
||||
const reading = (r.reading || []).map(l => `<div class="vv-au-find info">${_esc(l)}</div>`).join('');
|
||||
const ask = document.getElementById('vv-au-ai-chat')
|
||||
? '<div class="vv-au-modal-acts"><button class="vv-au-btn prim" id="vv-au-triage-ask">Ask the assistant</button></div>' : '';
|
||||
|
||||
return `<div class="vv-au-card vv-au-sim">
|
||||
<div class="vv-au-why-sec" style="margin-top:0;border-top:none;padding-top:0">
|
||||
${r.total} of the last ${r.files_read} certbot runs failed${r.unclassified ? ` · ${r.unclassified} matched nothing known` : ''}
|
||||
</div>
|
||||
${cats}${reading}${ask}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function _triageAsk(r) {
|
||||
const chat = window.__vvAiChat && window.__vvAiChat['vv-au-ai'];
|
||||
if (!chat) return;
|
||||
if (chat.busy()) { vvAlert('The assistant is still answering the previous question.'); return; }
|
||||
|
||||
const L = [];
|
||||
L.push(`Certificate renewals on this machine. ${r.total} of the last ${r.files_read} certbot runs failed. `
|
||||
+ `Categories, counted per run rather than per log line:`);
|
||||
for (const c of (r.categories || []))
|
||||
L.push(` ${c.count} — ${c.what}${c.root ? ' [root cause]' : ' [consequence]'}`
|
||||
+ ((c.domains || []).length ? ` — ${c.domains.join(', ')}` : ''));
|
||||
for (const l of (r.reading || [])) L.push(l);
|
||||
L.push('');
|
||||
L.push('What do I fix first, and in what order? Be concrete about which hostnames to add DNS for, '
|
||||
+ 'delete, or remove from NPM — and say whether the rate limit clears on its own once that is done.');
|
||||
|
||||
chat.retarget('troubleshoot', 'certificates', 'now looking at certificate renewals');
|
||||
const input = document.getElementById('vv-au-ai-input');
|
||||
if (input) input.value = L.join('\n');
|
||||
chat.send();
|
||||
const card = document.getElementById('vv-au-ai-card');
|
||||
if (card) card.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
|
||||
}
|
||||
|
||||
// ── Assistant ────────────────────────────────────────────────────────────────
|
||||
// vv_ai_chat_markup() emits markup and nothing else — the instance has to be constructed here,
|
||||
// as every other placement does. Without this the card renders complete and is entirely inert:
|
||||
|
||||
Reference in New Issue
Block a user