156 lines
7.8 KiB
PHP
156 lines
7.8 KiB
PHP
<?php
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
// PURPOSE
|
|
// Recall and precision for the syslog triage patterns. See ai_log_check.sh for why both
|
|
// halves exist and why neither alone is enough.
|
|
//
|
|
// OPERATIONAL MODEL
|
|
// Reads ai_log_fixtures.txt and this host's /var/log/syslog*. Files nothing, writes nothing,
|
|
// and calls no part of the sweep beyond vv_ai_syslog_findings() on lines it supplies itself.
|
|
//
|
|
// DESIGN PRINCIPLES
|
|
// Only recall can fail the run.
|
|
// A missed fixture is a fact about the patterns and is always a defect. A precision hit is
|
|
// a fact about this machine — a genuinely failing disk should not turn this red, and if it
|
|
// did, the honest fix would be to stop having a failing disk rather than to edit a pattern.
|
|
//
|
|
// Precision is replayed against real history, not a sample.
|
|
// The patterns that cause damage are the ones matching ordinary operation, and ordinary
|
|
// operation is exactly what a hand-written fixture file never contains. Only the machine's
|
|
// own syslog can show what a pattern fires on when nothing is wrong.
|
|
//
|
|
// The sweep is never invoked, only its matcher.
|
|
// vv_ai_syslog_findings() is called on lines this file supplies. Running the real sweep
|
|
// would file findings, and a test that has to be cleaned up afterwards stops being run.
|
|
//
|
|
// OPERATIONAL SAFEGUARDS
|
|
// Read-only. Reads ai_log_fixtures.txt and this host's /var/log/syslog*; files no finding,
|
|
// writes no store, and touches no conf beyond the enable flag.
|
|
//
|
|
// Exit 0 when every fixture is recognised as written. Precision findings are reported but
|
|
// never fail the run — see DESIGN PRINCIPLES.
|
|
//
|
|
// RUNTIME MODES
|
|
// php ai_log_check.php both checks
|
|
// php ai_log_check.php --recall fixtures only
|
|
// php ai_log_check.php --precision replay this host's syslog history only
|
|
//
|
|
// Not scheduled, and deliberately so. Run it after touching VV_AI_SYSLOG_PATTERNS.
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
require_once dirname(__DIR__) . '/include/ai_repair.php';
|
|
|
|
$args = array_slice($argv ?? [], 1);
|
|
$only = in_array('--precision', $args, true) ? 'precision'
|
|
: (in_array('--recall', $args, true) ? 'recall' : 'both');
|
|
$fixtures = __DIR__ . '/ai_log_fixtures.txt';
|
|
|
|
$pass = 0; $fail = 0;
|
|
function ok(string $what, bool $cond, string $got = ''): void {
|
|
global $pass, $fail;
|
|
if ($cond) { $pass++; printf(" ok %s\n", $what); }
|
|
else { $fail++; printf(" FAIL %s%s\n", $what, $got !== '' ? "\n → $got" : ''); }
|
|
}
|
|
|
|
// One line at a time, so a fixture is asserted on its own rather than on whatever aggregated
|
|
// with it. The finders are given the line directly, bypassing the file, the timestamp filter and
|
|
// docker — those are tested separately, and a fixture dated last August would otherwise be
|
|
// silently dropped for being older than the marker.
|
|
function classify(string $line, string $source = 'sys'): ?array {
|
|
if ($source === 'ctr') {
|
|
$f = vv_ai_container_findings(0, ['fixture-container' => [$line]]);
|
|
return $f[0] ?? null;
|
|
}
|
|
$f = vv_ai_syslog_findings(0, [$line]);
|
|
return $f[0] ?? null;
|
|
}
|
|
|
|
if ($only !== 'precision') {
|
|
echo "── recall: fixtures ──────────────────────────────────────────────\n";
|
|
if (!is_readable($fixtures)) {
|
|
echo " FAIL cannot read $fixtures\n";
|
|
exit(1);
|
|
}
|
|
foreach (file($fixtures, FILE_IGNORE_NEW_LINES) as $n => $raw) {
|
|
$line = rtrim($raw);
|
|
if ($line === '' || $line[0] === '#') continue;
|
|
|
|
// A ! line must match nothing at all. ! source | line
|
|
if ($line[0] === '!') {
|
|
$rest = trim(substr($line, 1));
|
|
$bits = explode('|', $rest, 2);
|
|
if (count($bits) !== 2) { ok(sprintf('L%d is malformed', $n + 1), false, $line); continue; }
|
|
$src = trim($bits[0]);
|
|
$sample = trim($bits[1]);
|
|
$got = classify($sample, $src);
|
|
ok(sprintf('L%-3d %s no match: %s', $n + 1, $src, mb_substr($sample, 0, 58)),
|
|
$got === null, $got ? "matched as {$got['ref']} / {$got['subject']}" : '');
|
|
continue;
|
|
}
|
|
|
|
$parts = explode('|', $line, 4);
|
|
if (count($parts) !== 4) { ok(sprintf('L%d is malformed', $n + 1), false, $line); continue; }
|
|
[$src, $wantSubject, $wantLevel, $sample] = array_map('trim', $parts);
|
|
|
|
$got = classify($sample, $src);
|
|
if ($got === null) {
|
|
ok(sprintf('L%-3d %s', $n + 1, mb_substr($sample, 0, 64)), false, 'no pattern matched');
|
|
continue;
|
|
}
|
|
ok(sprintf('L%-3d %-12s %-5s %s', $n + 1, $got['subject'], $got['sys_level'],
|
|
mb_substr($got['ref'], 0, 30)),
|
|
$got['subject'] === $wantSubject && $got['sys_level'] === $wantLevel,
|
|
sprintf('wanted %s/%s, got %s/%s', $wantSubject, $wantLevel,
|
|
$got['subject'], $got['sys_level']));
|
|
}
|
|
}
|
|
|
|
if ($only !== 'recall') {
|
|
echo "\n── precision: this host's real syslog history ────────────────────\n";
|
|
$total = 0; $hits = [];
|
|
foreach (glob('/var/log/syslog*') ?: [] as $file) {
|
|
if (!is_readable($file)) continue;
|
|
$fh = @fopen($file, 'r');
|
|
if (!$fh) continue;
|
|
while (($l = fgets($fh)) !== false) {
|
|
$total++;
|
|
$got = classify(rtrim($l));
|
|
if ($got === null) continue;
|
|
$k = $got['ref'] . ' | ' . $got['subject'] . ' | ' . $got['sys_level'];
|
|
$hits[$k] = ($hits[$k] ?? 0) + 1;
|
|
}
|
|
fclose($fh);
|
|
}
|
|
printf(" scanned %s lines\n", number_format($total));
|
|
if ($hits) { arsort($hits); foreach ($hits as $k => $n) printf(" %-7s %s\n", number_format($n), $k); }
|
|
else { echo " nothing matched\n"; }
|
|
|
|
// Every running container, read far deeper than a sweep ever does. A pattern that is quiet
|
|
// over this much real output is a pattern that will be quiet in service.
|
|
echo "\n── precision: every running container's log ──────────────────────\n";
|
|
$cTotal = 0; $hits = [];
|
|
$names = vv_ai_running_containers();
|
|
foreach ($names as $name) {
|
|
$lines = vv_ai_container_log($name, 1, 2000);
|
|
$cTotal += count($lines);
|
|
$found = vv_ai_container_findings(0, [$name => $lines]);
|
|
foreach ($found as $f) {
|
|
$k = $f['ref'] . ' | ' . $f['subject'] . ' | ' . $f['sys_level'];
|
|
$hits[$k] = ($hits[$k] ?? 0) + (int)filter_var($f['observed'], FILTER_SANITIZE_NUMBER_INT);
|
|
}
|
|
}
|
|
printf(" scanned %s lines across %d container%s\n", number_format($cTotal), count($names),
|
|
count($names) === 1 ? '' : 's');
|
|
if (!$hits) {
|
|
echo " nothing matched — this host is not reporting any of these faults\n";
|
|
} else {
|
|
arsort($hits);
|
|
foreach ($hits as $k => $n) printf(" %-7s %s\n", number_format($n), $k);
|
|
echo "\n Each line above is a fault the sweep would file. If any of them is normal\n"
|
|
. " operation on this machine, the pattern is wrong — add it to the fixtures as a\n"
|
|
. " ! line and tighten the pattern until it stops matching.\n";
|
|
}
|
|
}
|
|
|
|
printf("\n%d passed, %d failed\n", $pass, $fail);
|
|
exit($fail ? 1 : 0);
|