From 01601d210b932f1083ed1991c43a71b59e889646 Mon Sep 17 00:00:00 2001
From: Gmer4Lfe <gmer4lfe@gmail.com>
Date: Sun, 16 Aug 2026 01:59:11 -0400
Subject: [PATCH] Watch the auth stack on a schedule, so nobody has to open the
 tab

Both checks already answered their question on demand and both needed
somebody to press a button on the right row. One host here has returned
nothing but 5xx for months.

Filed as findings, which is the existing answer to a condition that
persists while nobody is looking. Grouped by cause rather than by
hostname: a default policy of bypass produced twenty-two findings that
were one sentence repeated, and they have one fix between them.
---
 Deployment/master.conf.template     |  39 ++++
 Plugin/unraid/Tools/auth_sweep.php  | 294 ++++++++++++++++++++++++++++
 Plugin/unraid/Tools/auth_sweep.sh   |  50 +++++
 Plugin/unraid/include/ai_repair.php |  14 ++
 4 files changed, 397 insertions(+)
 create mode 100644 Plugin/unraid/Tools/auth_sweep.php
 create mode 100755 Plugin/unraid/Tools/auth_sweep.sh

diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template
index f7058d4..9f4369a 100644
--- a/Deployment/master.conf.template
+++ b/Deployment/master.conf.template
@@ -494,6 +494,11 @@
         # per-domain renewal and failure counts.
         "Plugin/unraid/Tools/cert_history.sh"         # record cert renewals, failures and age per domain
         "Plugin/unraid/Tools/npm_access_stats.sh"    # aggregate NPM per-host access logs into request and byte totals
+        # After both of the above, because it reads what they write — the uptime history and the
+        # per-host request totals are two of the four things it reasons from. Daily rather than
+        # hourly: everything it files is a condition that has already lasted hours by the time
+        # AUTH_SWEEP_DOWN_MIN lets it through, so a faster cadence would find nothing new.
+        "Plugin/unraid/Tools/auth_sweep.sh"          # file findings for hosts that are not serving, and hostnames Authelia is not protecting
     )
 
 # Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart.
@@ -1498,6 +1503,40 @@
     UPTIME_PROBE_TIMEOUT=8      # seconds per domain before it counts as down
     UPTIME_PROBE_LIST_TTL=900   # seconds to reuse the domain list from NPM before re-reading it
 
+# ── Auth Sweep ──
+# Tools/auth_sweep.sh asks the two questions the Auth tab answers about one host, about every host,
+# and files what it finds as findings on the AI tab. Reports only — it starts nothing and rewrites
+# nothing, because every remedy here (start a container, edit a rule, change a default policy) is a
+# decision rather than a correction.
+#
+#   proxy_down   a host below UPTIME_MIN that has been failing longer than DOWN_MIN. The time gate
+#                is what keeps a reboot from filing a finding for every hostname on the machine.
+#   access_open  an Authelia instance whose default policy lets through every hostname its rules do
+#                not decide. Filed once per instance, not once per hostname — they all have the
+#                same single fix, and one finding per name is twenty-two copies of one sentence.
+#
+# The access half reads whichever Authelia each proxy host actually points at, which is not always
+# the one HOST*_AUTHELIA_CONFIG names — this installation runs two.
+    AUTH_SWEEP_ENABLED=true      # master switch
+    AUTH_SWEEP_UPTIME_MIN=96     # 24h percentage below which a host becomes a candidate
+    AUTH_SWEEP_DOWN_MIN=120      # minutes it must have been failing before anything is filed
+    AUTH_SWEEP_ACCESS_CHECK=true # run the "is it actually protected" half at all
+
+# ── Cert Triage ──
+# Tools/cert_triage.sh reads certbot's own logs and names why renewals failed. cert_history.sh
+# counts failures by noticing an expiry in the past; this reads the reason.
+#
+# Counts runs, not lines: one log file is one certbot invocation, and one failure writes its
+# reason into the ACME response, the traceback and certbot's summary, so line counting reports it
+# three times and inflates whichever category is most verbose.
+#
+# The log directory is found from the NPM container's own mount. Set CERT_TRIAGE_LOG_DIR only if
+# that lookup cannot work. Both bounds exist because this is reachable from a page request and the
+# directory here is 639 MB across a thousand rotated files.
+    CERT_TRIAGE_FILES=40         # rotated logs to read, newest first by rotation suffix
+    CERT_TRIAGE_MAX_BYTES=262144 # bytes read from the end of each — a run's reason is always last
+    CERT_TRIAGE_LOG_DIR=""       # empty = find it from the NPM container
+
 # ━━━ Backup Verify ━━━
 # Verifies rsync mirror health by comparing random file checksums between servers.
 # Catches silent corruption or incomplete syncs that rsync itself wouldn't detect.
diff --git a/Plugin/unraid/Tools/auth_sweep.php b/Plugin/unraid/Tools/auth_sweep.php
new file mode 100644
index 0000000..29616a9
--- /dev/null
+++ b/Plugin/unraid/Tools/auth_sweep.php
@@ -0,0 +1,294 @@
+<?php
+// ═══════════════════════════════════════════════════════════════════════════════════════════════
+// PURPOSE
+//   Looks at the whole auth stack the way the Auth tab's two checks look at one thing, and files
+//   what it finds. Two questions, asked of every host NPM serves:
+//
+//     Is this host actually serving?            — the why-check, for anything below threshold
+//     Is this host actually protected?          — the access check, for anything behind Authelia
+//
+// WHY IT EXISTS
+//   Both answers already existed on demand, and both required somebody to open the Auth tab and
+//   press a button on the right row. Nobody does that on a working day. One host here has returned
+//   nothing but 5xx to every request for months, and the hostnames that sit behind an auth_request
+//   block which then waves everyone through are invisible from every page in the plugin, because
+//   the fact is split across NPM, an Authelia config and the directory.
+//
+//   Findings are the existing answer to "a condition that persists and nobody is looking" — the AI
+//   tab already lists, grades, ages out and notifies on them. This adds the auth stack as a source
+//   rather than inventing a second place for the same idea.
+//
+// OPERATIONAL MODEL
+//   One pass = read NPM once, then per host: the recorded uptime, and — only where it is warranted
+//   — the live checks. The expensive part is deliberately gated:
+//
+//     proxy_down   filed when 24h uptime is below AUTH_SWEEP_UPTIME_MIN *and* the host has been in
+//                  that state longer than AUTH_SWEEP_DOWN_MIN. A restart or a reboot produces a
+//                  perfectly ordinary dip, and a finding for every one of those is a list nobody
+//                  reads. Live checks run only for hosts that pass this gate.
+//
+//     access_open  filed when a host carries an auth_request block and the Authelia deciding it
+//                  reaches its default policy of bypass for a member of no relevant group. No
+//                  network calls at all — this is three config files compared.
+//
+//   Nothing is repaired, restarted or rewritten. This is a reporting pass, and it stays one: the
+//   remedies here are "start a container", "edit a rule", "change a default policy", and every one
+//   of them is a decision rather than a correction.
+//
+// DESIGN PRINCIPLES
+//   The gate is time, not count.
+//       A host that has been down for four hours is news whether the probe caught eight samples
+//       or eight hundred. Basing it on samples would file findings at a different threshold on a
+//       node whose probe runs at a different cadence.
+//
+//   Findings are refreshed, not duplicated.
+//       vv_ai_finding_write() keys on kind + subject + ref, so a host still down tomorrow updates
+//       the record it filed today rather than adding a second one. The store closes what stops
+//       recurring on its own.
+//
+//   The access check answers for a real user, not a hypothetical one.
+//       "Does the default policy let somebody in" is only meaningful about a person who exists.
+//       The sweep asks about a directory member holding no privileged group, because that is the
+//       account that reveals a rule which protects nothing.
+//
+// OPERATIONAL SAFEGUARDS
+//   Non-fatal, always. No NPM, no credentials, no domains, AI repair switched off — exits 0.
+//   One pass at a time, flock non-blocking, so a slow pass cannot overlap the next.
+//   Live probing is bounded by the gate above, so a total outage cannot turn one pass into
+//   thirty-five sequential timeouts.
+//
+// RUNTIME MODES
+//   auth_sweep.php             one pass, files findings
+//   auth_sweep.php --dry-run   report what it would file, write nothing
+//   auth_sweep.php --report    one-screen summary for the Sunday report; silent when clean
+//
+// CONFIGURATION
+//   AUTH_SWEEP_ENABLED       master switch (default true)
+//   AUTH_SWEEP_UPTIME_MIN    24h percentage below which a host is a candidate (default 96)
+//   AUTH_SWEEP_DOWN_MIN      minutes it must have been failing before filing (default 120)
+//   AUTH_SWEEP_ACCESS_CHECK  whether to run the protection half at all (default true)
+// ═══════════════════════════════════════════════════════════════════════════════════════════════
+
+require_once dirname(__DIR__) . '/include/auth.php';
+require_once dirname(__DIR__) . '/include/ai_repair.php';
+
+$dryRun = in_array('--dry-run', $argv, true);
+$report = in_array('--report',  $argv, true);
+
+$lock = @fopen(sys_get_temp_dir() . '/vv_auth_sweep.lock', 'c');
+if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) { echo "another pass is running\n"; exit(0); }
+
+try {
+    $v = vv_conf_vars();
+    if (strtolower(trim($v['AUTH_SWEEP_ENABLED'] ?? 'true')) === 'false') {
+        echo "AUTH_SWEEP_ENABLED is false\n"; exit(0);
+    }
+    $minPct   = (float) ($v['AUTH_SWEEP_UPTIME_MIN'] ?? 96);
+    $minDown  = max(1, (int) ($v['AUTH_SWEEP_DOWN_MIN'] ?? 120)) * 60;
+    $doAccess = strtolower(trim($v['AUTH_SWEEP_ACCESS_CHECK'] ?? 'true')) !== 'false';
+
+    $p = vv_npm_list_proxies();
+    if (!($p['ok'] ?? false)) { echo 'NPM: ' . ($p['error'] ?? 'unreadable') . "\n"; exit(0); }
+
+    $uptime = is_file(vv_auth_db_file('uptime.json'))
+            ? (json_decode((string) @file_get_contents(vv_auth_db_file('uptime.json')), true) ?: []) : [];
+    $doms   = $uptime['domains'] ?? [];
+
+    $filed = $skipped = 0;
+    $lines = [];
+
+    // ── Half one: hosts that are not serving ──
+    foreach ($p['proxies'] as $h) {
+        // A host switched off is not a fault, it is a decision. Filing one would put a finding on
+        // the list for every host the operator has deliberately parked.
+        if (($h['enabled'] ?? true) === false) continue;
+
+        $names = [];
+        foreach ($h['domain_names'] ?? [] as $d) {
+            $d = strtolower(trim((string) $d));
+            if ($d !== '' && !str_contains($d, '*')) $names[] = $d;
+        }
+        if (!$names) continue;
+
+        // The worst name decides, for the same reason the row does: a host is only as reachable as
+        // its least reachable hostname, and averaging hides one dead name behind three healthy ones.
+        $worstPct = null; $worstDom = ''; $worstRec = null;
+        foreach ($names as $d) {
+            $r = $doms[$d] ?? null;
+            if (!$r) continue;
+            $pct = vv_auth_uptime_window($r['hours'] ?? [], 24);
+            if ($pct === null) continue;
+            if ($worstPct === null || $pct < $worstPct) { $worstPct = $pct; $worstDom = $d; $worstRec = $r; }
+        }
+        if ($worstPct === null || $worstPct >= $minPct) continue;
+
+        // How long it has actually been like this. last_change is when the state last flipped, so
+        // a host that went down two minutes ago is excluded here and caught on a later pass — which
+        // is the whole point of the gate.
+        $since = (int) ($worstRec['last_change'] ?? 0);
+        $for   = $since > 0 ? time() - $since : 0;
+        if (($worstRec['state'] ?? '') === 'down' && $for < $minDown) { $skipped++; continue; }
+
+        $why = vv_npm_why((int) ($h['id'] ?? 0));
+        if (!($why['ok'] ?? false)) continue;
+
+        // The findings the check already writes, which is the whole reason this does not have its
+        // own opinion about what is wrong. Only the decisive ones are carried into the record.
+        $said = [];
+        foreach ($why['findings'] as $f) if (in_array($f['level'], ['bad', 'warn'], true)) $said[] = $f['text'];
+        if (!$said) { $skipped++; continue; }
+
+        $evidence = sprintf("%s is at %.2f%% over 24h%s.\n\n%s",
+            $worstDom, $worstPct,
+            $for > 0 ? ' and has been ' . ($worstRec['state'] ?? 'failing') . ' for ' . round($for / 3600, 1) . ' hours' : '',
+            implode("\n", $said));
+
+        $lines[] = sprintf('  %-34s %6.2f%%  %s', $worstDom, $worstPct, $said[0]);
+        if ($dryRun) { $filed++; continue; }
+
+        $w = vv_ai_finding_write([
+            'kind'     => 'proxy_down',
+            'subject'  => implode(', ', $names),
+            'ref'      => 'npm:proxy:' . ($h['id'] ?? 0),
+            'evidence' => $evidence,
+            'observed' => sprintf('%.2f%% over 24h', $worstPct),
+            // Proven, because these are measurements rather than an inference: a TCP connect either
+            // completed or it did not, and the access log either counted 5xx or it did not.
+            'proven'   => true,
+        ]);
+        $w['ok'] ? $filed++ : $skipped++;
+    }
+
+    // ── Half two: hosts that are guarded but not protected ──
+    //
+    // Grouped by Authelia instance, not filed per hostname. The first version of this produced
+    // twenty-two findings that were all the same sentence, because they all had the same cause: a
+    // default policy of bypass means every host whose rule does not name your group lets you
+    // through, so the number of findings was really the number of hostnames. One finding per
+    // instance, naming the hosts it affects, is the fact — and it has one fix rather than
+    // twenty-two.
+    $openLines = [];
+    if ($doAccess) {
+        // Somebody who exists and holds none of the groups the rules name. A rule that still lets
+        // this account through is a rule protecting nothing, and asking about an invented username
+        // would prove nothing about the directory.
+        $probe = vv_auth_sweep_ordinary_user();
+        $byInstance = [];
+
+        foreach ($p['proxies'] as $h) {
+            if (($h['enabled'] ?? true) === false) continue;
+            if (!str_contains((string) ($h['advanced_config'] ?? ''), 'auth_request')) continue;
+
+            foreach ($h['domain_names'] ?? [] as $d) {
+                $d = strtolower(trim((string) $d));
+                if ($d === '' || str_contains($d, '*')) continue;
+
+                $a = vv_auth_access_check($d, $probe);
+                if (!($a['ok'] ?? false)) continue;
+                if (($a['policy'] ?? '') !== 'bypass') continue;
+
+                $inst = (string) ($a['authelia']['container'] ?? '?');
+                $byInstance[$inst]['default'] = (string) ($a['default_policy'] ?? '?');
+                $byInstance[$inst]['config']  = (string) ($a['authelia']['config'] ?? '');
+                // Which of the two shapes this is, per host: a hostname no rule mentions, or one a
+                // rule covers and then steps over. They have different fixes — write a rule, or
+                // widen an existing one — so the record keeps them apart.
+                $stepped = false;
+                foreach ($a['trace'] ?? [] as $t) if (($t['skip'] ?? '') === 'subject') $stepped = true;
+                $byInstance[$inst][$stepped ? 'stepped' : 'unlisted'][] = $d;
+            }
+        }
+
+        foreach ($byInstance as $inst => $g) {
+            $unlisted = $g['unlisted'] ?? [];
+            $stepped  = $g['stepped']  ?? [];
+            $all      = array_merge($unlisted, $stepped);
+            if (!$all) continue;
+
+            $ev = [];
+            $ev[] = $inst . ' has default_policy: ' . ($g['default'] ?? '?') . ', so any request its rules do '
+                  . 'not decide is allowed through. ' . count($all) . ' hostname'
+                  . (count($all) === 1 ? '' : 's') . ' behind an auth_request block pointing at it reach the '
+                  . 'application without being asked to authenticate.';
+            if ($unlisted) $ev[] = "\nNo rule mentions these at all:\n  " . implode("\n  ", $unlisted);
+            if ($stepped)  $ev[] = "\nA rule covers these but does not apply to an ordinary account"
+                                 . ($probe ? ' (tested as ' . $probe . ')' : '') . ":\n  " . implode("\n  ", $stepped);
+            $ev[] = "\nRules are in " . ($g['config'] ?: 'a config that was not found') . '.';
+            // Said plainly because "bypass" reads as harmless and it is the single most
+            // consequential line in that file.
+            $ev[] = "\nThe fix is a default_policy of deny with an explicit rule for anything that is "
+                  . "meant to be public — not a rule per hostname above.";
+
+            $openLines[] = sprintf('  %-22s default bypass — %d hostname%s unprotected',
+                                   $inst, count($all), count($all) === 1 ? '' : 's');
+            if ($dryRun) { $filed++; continue; }
+
+            $w = vv_ai_finding_write([
+                'kind'     => 'access_open',
+                'subject'  => $inst . ' — ' . count($all) . ' hostnames not protected',
+                'ref'      => 'authelia:' . $inst . ':default_policy',
+                'evidence' => implode("\n", $ev),
+                'observed' => 'default_policy: ' . ($g['default'] ?? '?'),
+                'proven'   => true,
+            ]);
+            $w['ok'] ? $filed++ : $skipped++;
+        }
+    }
+
+    if ($report) {
+        // Silent on a clean week. The orchestrator's job is to say nothing when there is nothing
+        // to say, and a section that always prints is a section that stops being read.
+        if (!$lines && !$openLines) exit(0);
+        echo "Auth stack review\n";
+        if ($lines)     { echo "\nProxy hosts not serving:\n";        foreach ($lines as $l)     echo "$l\n"; }
+        if ($openLines) { echo "\nBehind Authelia but not protected:\n"; foreach ($openLines as $l) echo "$l\n"; }
+        exit(1);
+    }
+
+    printf("%s%d finding%s, %d skipped\n", $dryRun ? 'dry run — ' : '', $filed, $filed === 1 ? '' : 's', $skipped);
+    foreach (array_merge($lines, $openLines) as $l) echo "$l\n";
+    exit(0);
+
+} finally {
+    flock($lock, LOCK_UN);
+    fclose($lock);
+}
+
+// A directory member holding none of the groups any rule names. Returns '' when every user is
+// privileged or the directory cannot be read, and the caller then asks about no user at all —
+// which still answers the "no rule mentions this host" case and simply cannot answer the
+// "the rule stepped over this person" one.
+function vv_auth_sweep_ordinary_user(): string {
+    $named = [];
+    // Groups named by the rules of every Authelia instance in play, not just the configured one —
+    // a .us hostname is decided by a config this conf file does not point at.
+    foreach (vv_auth_sweep_configs() as $cfg) {
+        $r = vv_authelia_read_rules($cfg);
+        foreach (($r['ok'] ?? false) ? $r['rules'] : [] as $rule) {
+            $s = $rule['subject'] ?? null;
+            foreach (is_array($s) ? $s : [$s] as $alt)
+                foreach (is_array($alt) ? $alt : [$alt] as $one)
+                    if (is_string($one) && str_starts_with($one, 'group:')) $named[strtolower(substr($one, 6))] = true;
+        }
+    }
+
+    $u = vv_lldap_list_users();
+    foreach (($u['ok'] ?? false) ? $u['users'] : [] as $user) {
+        $mine = array_map('strtolower', array_filter(array_column($user['groups'] ?? [], 'displayName')));
+        if (array_intersect($mine, array_keys($named))) continue;
+        return (string) ($user['id'] ?? '');
+    }
+    return '';
+}
+
+// Every Authelia configuration this installation actually uses, discovered through the proxy hosts
+// rather than listed anywhere. Two instances run here and conf names one.
+function vv_auth_sweep_configs(): array {
+    $out = [];
+    $p = vv_npm_list_proxies();
+    foreach (($p['ok'] ?? false) ? $p['proxies'] : [] as $h) {
+        $i = vv_authelia_instance_for($h);
+        if (($i['config'] ?? '') !== '' && is_file($i['config'])) $out[$i['config']] = true;
+    }
+    return array_keys($out);
+}
diff --git a/Plugin/unraid/Tools/auth_sweep.sh b/Plugin/unraid/Tools/auth_sweep.sh
new file mode 100755
index 0000000..b587e67
--- /dev/null
+++ b/Plugin/unraid/Tools/auth_sweep.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+# ==============================================================================================
+# ===================================== Auth Sweep =============================================
+# ==============================================================================================
+#
+# PURPOSE
+# ─────────────────────────────────────────────────────────────────────────────
+# Asks the two questions the Auth tab can answer about one host, about every host, and files
+# what it finds as findings.
+#
+#   Is this host serving?    below the uptime threshold, and failing for longer than a restart
+#   Is this host protected?  behind an auth_request block that no rule then applies to
+#
+# Both answers existed already and both needed somebody to open the tab and press a button on
+# the right row. One host here has returned nothing but 5xx for months.
+#
+# ==============================================================================================
+# OPERATIONAL MODEL
+# ==============================================================================================
+#
+# A wrapper. The work is in auth_sweep.php.
+#
+# Reports only — nothing is started, restarted or rewritten. The remedies are "start a
+# container", "edit a rule", "change a default policy", and each of those is a decision.
+#
+# The live half is gated on time rather than on sample count: a host must have been failing for
+# longer than AUTH_SWEEP_DOWN_MIN before anything is filed, so a reboot does not produce a
+# finding for every hostname on the machine.
+#
+# ==============================================================================================
+# RUNTIME MODES
+# ==============================================================================================
+#
+# auth_sweep.sh             one pass, files findings
+# auth_sweep.sh --dry-run   report what it would file, write nothing
+# auth_sweep.sh --report    one-screen summary for the Sunday report; silent when clean
+#
+# ==============================================================================================
+# CONFIGURATION
+# ==============================================================================================
+#
+# AUTH_SWEEP_ENABLED       master switch
+# AUTH_SWEEP_UPTIME_MIN    24h percentage below which a host is a candidate
+# AUTH_SWEEP_DOWN_MIN      minutes it must have been failing before a finding is filed
+# AUTH_SWEEP_ACCESS_CHECK  whether to run the protection half at all
+#
+# ==============================================================================================
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+php "$SCRIPT_DIR/auth_sweep.php" "$@"
diff --git a/Plugin/unraid/include/ai_repair.php b/Plugin/unraid/include/ai_repair.php
index f4a593f..3f2d0b9 100644
--- a/Plugin/unraid/include/ai_repair.php
+++ b/Plugin/unraid/include/ai_repair.php
@@ -92,6 +92,13 @@ const VV_AI_FINDING_KINDS = [
     'container_fault' => 'a container is logging a fault about its own environment',
     'media_misfiled'  => 'a series is shelved somewhere its own metadata does not support',
     'watchdog_strike' => 'a watchdog has counted something far enough to be worth a record',
+    // The auth stack's two. Neither is a conf problem — one is a service behind a proxy host that
+    // has stopped answering, the other is a hostname put behind Authelia that Authelia then lets
+    // everyone past. They are recorded here rather than only drawn on the Auth tab because both
+    // are conditions that persist for weeks without anyone opening that tab: one host on this
+    // installation has served nothing but 5xx for months.
+    'proxy_down'      => 'a proxy host has been failing long enough that it is not a blip',
+    'access_open'     => 'a hostname behind an auth_request block is not actually protected by any rule',
 ];
 
 // Which kinds are a statement about Varaverk's configuration, and which are a statement about
@@ -205,6 +212,13 @@ function vv_ai_finding_severity(array $f): string {
         'missing_value'  => 'error',   // configured to use something that was never supplied
         'unreachable'    => 'warn',    // may be transient; the strike system is what escalates it
         'unknown_target' => 'warn',
+        // A hostname deliberately put behind authentication that authenticates nobody is the one
+        // finding here that is worse the longer it goes unnoticed, and it is never transient —
+        // it is a state of the configuration, not a passing failure.
+        'access_open'    => 'error',
+        // The sweep only files this after the outage has already outlasted a restart, so it is
+        // past the point where the strike system would still be deciding.
+        'proxy_down'     => 'error',
         default          => 'warn',
     };
 }