Gmer4Lfe
987313e7dc
Writing down what each endpoint actually guarantees made the places it didn't obvious — shell arguments reaching a crontab or a bash -c unescaped, master.conf written without tmp+rename, and conf edits that could be saved without ever being parsed.
214 lines
11 KiB
PHP
214 lines
11 KiB
PHP
<?php
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
// PURPOSE
|
|
// Attention board. The three things worth knowing without opening a tab: locks nobody is
|
|
// holding, scripts whose last run ended badly, and whether the partner is reachable.
|
|
//
|
|
// OPERATIONAL MODEL
|
|
// A dashboard of exceptions, not of state. Everything here is designed to be empty on a
|
|
// healthy host — an empty board is the expected reading, which is what makes a non-empty
|
|
// one worth looking at.
|
|
//
|
|
// Each section answers a question no single other endpoint does. Locks come from the
|
|
// filesystem, errors from correlating logs against their stat files, reachability from an
|
|
// actual ping. They are collected together because they are read together.
|
|
//
|
|
// DESIGN PRINCIPLES
|
|
// Only stale locks are reported.
|
|
// A lock whose owning pid is still in /proc is a job legitimately running and is
|
|
// skipped. What remains is the set a person might need to clear — which is exactly what
|
|
// clearlock.php exists for.
|
|
//
|
|
// Errors are gated on the stat file, not on the log text.
|
|
// A script is only reported when its last recorded run exited warn or error. Scanning
|
|
// logs for the word "error" produced two persistent false positives: dry runs, which
|
|
// print failures they did not cause, and success summaries containing lines like
|
|
// "Failed: 0". The stat file is the run's own verdict, and it is authoritative.
|
|
//
|
|
// A script with no stat file is skipped entirely.
|
|
// No stat file means it never ran through run_job.sh, so there is no verdict to trust
|
|
// and no basis for reporting it.
|
|
//
|
|
// The error line is the last matching one, searched backwards.
|
|
// Logs are appended, so the most recent failure is at the end. The search walks up from
|
|
// there and stops at the first hit rather than reading forward and reporting the oldest.
|
|
//
|
|
// There is a fallback when nothing matches.
|
|
// A run that failed without printing a recognisable error still reports its last
|
|
// non-blank line. A script marked error with no explanation is worse than an imperfect
|
|
// one.
|
|
//
|
|
// The partner is discovered, not configured.
|
|
// master.conf is scanned for HOST<n> entries and the first non-self one is used, so this
|
|
// works for any number of hosts without a second list to maintain.
|
|
//
|
|
// Tailscale resolves the address, never local DNS.
|
|
// vv_resolve_tailscale_ip() mirrors common.sh, so the board tests the same path the
|
|
// rest of the system uses and survives the partner's IP changing.
|
|
//
|
|
// OPERATIONAL SAFEGUARDS
|
|
// Read-only. Nothing here clears a lock, truncates a log, or restarts anything — the board
|
|
// reports; clearlock.php and stop.php act.
|
|
//
|
|
// The lock scan is bounded to a hardcoded directory.
|
|
// glob over /tmp/unraid_locks/*.lock — a literal, not a config value, so no conf edit
|
|
// can point this scan somewhere else.
|
|
//
|
|
// The log walk is wrapped in a try/catch.
|
|
// RecursiveDirectoryIterator throws when LOG_DIR is absent or a subdirectory is
|
|
// unreadable — the normal state on a fresh install. The catch yields an empty error
|
|
// list rather than a 500.
|
|
//
|
|
// Every file read is independently suppressed and defaulted.
|
|
// @file_get_contents, @file, and ?: fallbacks throughout. One unreadable log or stat
|
|
// file costs its own row, not the response.
|
|
//
|
|
// The scan window and the output are both bounded.
|
|
// Logs older than 7 days are skipped, only the last 200 lines of each are read, each
|
|
// line is truncated at 220 characters, and the result is capped at 20 errors. This runs
|
|
// against a directory that grows without limit.
|
|
//
|
|
// ANSI escapes are stripped before matching and before returning.
|
|
// Logs are written with colour. Without stripping, the patterns would miss coloured
|
|
// error markers and the JSON would carry terminal control codes into the page.
|
|
//
|
|
// The ping target is escaped, and time-boxed by ping itself.
|
|
// escapeshellarg() on a value that came from conf, and -c1 -W2 so an unreachable
|
|
// partner costs two seconds. The result is cached 30s so the board's poll does not ping
|
|
// on every request.
|
|
//
|
|
// Reachability falls back to the hostname when Tailscale cannot resolve.
|
|
// Better to test something and report the result than to report nothing because the
|
|
// preferred resolution path failed.
|
|
//
|
|
// REQUEST
|
|
// GET, no parameters
|
|
//
|
|
// RESPONSE
|
|
// {"ok":true,
|
|
// "locks":[{"name","file","age"}], stale locks only — empty is healthy
|
|
// "errors":[{"script","line","ts"}], newest first, max 20
|
|
// "partner":{"host","reachable","latency"}} null when no partner is configured
|
|
//
|
|
// DEPENDS ON
|
|
// include/config.php LOG_DIR, vv_conf_vars(), vv_get_hostname(),
|
|
// vv_resolve_tailscale_ip(), vv_cache_read(), vv_cache_write()
|
|
// /tmp/unraid_locks lock files written by common.sh's locking helper
|
|
// LOG_DIR/** .log files and their .json stat files, written by run_job.sh
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
header('Content-Type: application/json');
|
|
require_once dirname(__DIR__) . '/include/config.php';
|
|
|
|
$out = ['ok' => true];
|
|
|
|
// ── Active locks ──────────────────────────────────────────────────────────
|
|
$lockDir = '/tmp/unraid_locks';
|
|
$locks = [];
|
|
if (is_dir($lockDir)) {
|
|
foreach (glob($lockDir . '/*.lock') ?: [] as $lf) {
|
|
$name = basename($lf, '.lock');
|
|
$age = time() - (int)filemtime($lf);
|
|
$content = trim(file_get_contents($lf) ?: '');
|
|
// content is "PID:scriptname" — extract PID
|
|
$pid = preg_match('/^(\d+)/', $content, $pm) ? $pm[1] : '';
|
|
// Skip if PID is still alive (it's legitimately running)
|
|
if ($pid && file_exists("/proc/$pid")) continue;
|
|
$locks[] = ['name' => $name, 'file' => basename($lf), 'age' => $age];
|
|
}
|
|
}
|
|
$out['locks'] = $locks;
|
|
|
|
// ── Recent errors ──────────────────────────────────────────────────────────
|
|
// Scan recursively so subdirectory logs (Orchestrators/, Media/, etc.) are included.
|
|
// e.script = relative path without extension = job ID base, e.g. "Orchestrators/transcode_management"
|
|
$errors = [];
|
|
if (is_dir(LOG_DIR)) {
|
|
$cutoff = time() - 7 * 86400;
|
|
$logBase = rtrim(LOG_DIR, '/') . '/';
|
|
try {
|
|
$ri = new RecursiveIteratorIterator(
|
|
new RecursiveDirectoryIterator(LOG_DIR, RecursiveDirectoryIterator::SKIP_DOTS)
|
|
);
|
|
foreach ($ri as $lf) {
|
|
if ($lf->getExtension() !== 'log') continue;
|
|
if ($lf->getMTime() < $cutoff) continue;
|
|
$rel = ltrim(str_replace($logBase, '', $lf->getPathname()), '/');
|
|
$script = preg_replace('/\.log$/', '', $rel);
|
|
|
|
// Only report scripts whose last recorded run exited as warn or error.
|
|
// This eliminates dry-run false positives (exit 0 = ok) and success-run
|
|
// summaries that happen to contain words like "Failed: 0".
|
|
$statFile = $logBase . $script . '.json';
|
|
if (file_exists($statFile)) {
|
|
$stat = json_decode(@file_get_contents($statFile), true);
|
|
$status = $stat['status'] ?? '';
|
|
if ($status !== 'warn' && $status !== 'error') continue;
|
|
} else {
|
|
continue; // no stat file — never ran through run_job.sh, skip
|
|
}
|
|
|
|
$lines = array_slice(@file($lf->getPathname()) ?: [], -200);
|
|
$lastErr = null;
|
|
foreach (array_reverse($lines) as $raw) {
|
|
$clean = preg_replace('/\033\[[0-9;]*[mK]/', '', rtrim($raw));
|
|
if (!$clean) continue;
|
|
if (preg_match('/\[(?:ERROR|WARN|CRITICAL|FAILED)\]/i', $clean) ||
|
|
preg_match('/\b(?:ERROR|CRITICAL):\s/i', $clean) ||
|
|
str_contains($clean, '✗') ||
|
|
(str_contains($clean, '⚠') && !str_contains($clean, '♥'))) {
|
|
$lastErr = mb_substr($clean, 0, 220);
|
|
break;
|
|
}
|
|
}
|
|
// Fall back to last non-blank line if no error pattern found in log
|
|
if ($lastErr === null) {
|
|
foreach (array_reverse($lines) as $raw) {
|
|
$clean = preg_replace('/\033\[[0-9;]*[mK]/', '', rtrim($raw));
|
|
if ($clean && !str_starts_with($clean, '──')) { $lastErr = mb_substr($clean, 0, 220); break; }
|
|
}
|
|
}
|
|
if ($lastErr !== null)
|
|
$errors[] = ['script' => $script, 'line' => $lastErr, 'ts' => (int)$lf->getMTime()];
|
|
}
|
|
} catch (Exception $e) {}
|
|
usort($errors, fn($a, $b) => $b['ts'] - $a['ts']);
|
|
}
|
|
$out['errors'] = array_slice($errors, 0, 20);
|
|
|
|
// ── Partner reachability ───────────────────────────────────────────────────
|
|
$partnerData = vv_cache_read('board_partner', 30);
|
|
|
|
if (!$partnerData) {
|
|
// Discover partner hostname dynamically from master.conf (works for any number of hosts)
|
|
$vars = vv_conf_vars();
|
|
$mine = vv_get_hostname();
|
|
$partnerHost = null;
|
|
foreach ($vars as $k => $v) {
|
|
if (preg_match('/^HOST\d+$/', $k) && $v !== '' && strcasecmp($v, $mine) !== 0) {
|
|
$partnerHost = $v; // first non-self host is the partner
|
|
break;
|
|
}
|
|
}
|
|
|
|
if ($partnerHost) {
|
|
// Resolve via Tailscale — mirrors common.sh resolve_tailscale_ip().
|
|
// Uses `tailscale ip -4` so it survives IP changes; never relies on local DNS.
|
|
$partnerIp = vv_resolve_tailscale_ip($partnerHost);
|
|
$target = $partnerIp ?: $partnerHost;
|
|
$start = microtime(true);
|
|
$result = shell_exec('ping -c1 -W2 ' . escapeshellarg($target) . ' 2>&1');
|
|
$elapsed = (int)round((microtime(true) - $start) * 1000);
|
|
$reached = str_contains((string)$result, '1 received')
|
|
|| str_contains((string)$result, '1 packets received');
|
|
$partnerData = [
|
|
'host' => $partnerHost,
|
|
'reachable' => $reached,
|
|
'latency' => $reached ? $elapsed : null,
|
|
];
|
|
vv_cache_write('board_partner', $partnerData);
|
|
}
|
|
}
|
|
$out['partner'] = $partnerData;
|
|
|
|
echo json_encode($out);
|