- FallBack tab: per-node tier inventory + active fallback card with duration, tier, handback strikes, running container status - Watchdog tab: live system health (RAM bar + thresholds, load, uptime, daemon), docker watchdog strikes + skip list + restart history, stability strikes + reboot log, resource pressure alert card, config inventory (mem limits, required, pause/stop lists) - Swapped partnership/arrs tab order; FallBack between partnership and watchdog - Plugin source tree moved from Plugin/usr/local/emhttp/plugins/varaverk/ to Plugin/unraid/ - Deployment/ conf templates added
122 lines
5.6 KiB
PHP
122 lines
5.6 KiB
PHP
<?php
|
|
// Live board data: locks, recent errors, partner reachability.
|
|
header('Content-Type: application/json');
|
|
require_once dirname(__DIR__) . '/include/config.php';
|
|
|
|
$out = ['ok' => true];
|
|
|
|
// ── Active locks ──────────────────────────────────────────────────────────
|
|
$lockDir = '/tmp/unraid_locks';
|
|
$locks = [];
|
|
if (is_dir($lockDir)) {
|
|
foreach (glob($lockDir . '/*.lock') ?: [] as $lf) {
|
|
$name = basename($lf, '.lock');
|
|
$age = time() - (int)filemtime($lf);
|
|
$content = trim(file_get_contents($lf) ?: '');
|
|
// content is "PID:scriptname" — extract PID
|
|
$pid = preg_match('/^(\d+)/', $content, $pm) ? $pm[1] : '';
|
|
// Skip if PID is still alive (it's legitimately running)
|
|
if ($pid && file_exists("/proc/$pid")) continue;
|
|
$locks[] = ['name' => $name, 'file' => basename($lf), 'age' => $age];
|
|
}
|
|
}
|
|
$out['locks'] = $locks;
|
|
|
|
// ── Recent errors ──────────────────────────────────────────────────────────
|
|
// Scan recursively so subdirectory logs (Orchestrators/, Media/, etc.) are included.
|
|
// e.script = relative path without extension = job ID base, e.g. "Orchestrators/transcode_management"
|
|
$errors = [];
|
|
if (is_dir(LOG_DIR)) {
|
|
$cutoff = time() - 7 * 86400;
|
|
$logBase = rtrim(LOG_DIR, '/') . '/';
|
|
try {
|
|
$ri = new RecursiveIteratorIterator(
|
|
new RecursiveDirectoryIterator(LOG_DIR, RecursiveDirectoryIterator::SKIP_DOTS)
|
|
);
|
|
foreach ($ri as $lf) {
|
|
if ($lf->getExtension() !== 'log') continue;
|
|
if ($lf->getMTime() < $cutoff) continue;
|
|
$rel = ltrim(str_replace($logBase, '', $lf->getPathname()), '/');
|
|
$script = preg_replace('/\.log$/', '', $rel);
|
|
|
|
// Only report scripts whose last recorded run exited as warn or error.
|
|
// This eliminates dry-run false positives (exit 0 = ok) and success-run
|
|
// summaries that happen to contain words like "Failed: 0".
|
|
$statFile = $logBase . $script . '.json';
|
|
if (file_exists($statFile)) {
|
|
$stat = json_decode(@file_get_contents($statFile), true);
|
|
$status = $stat['status'] ?? '';
|
|
if ($status !== 'warn' && $status !== 'error') continue;
|
|
} else {
|
|
continue; // no stat file — never ran through run_job.sh, skip
|
|
}
|
|
|
|
$lines = array_slice(@file($lf->getPathname()) ?: [], -200);
|
|
$lastErr = null;
|
|
foreach (array_reverse($lines) as $raw) {
|
|
$clean = preg_replace('/\033\[[0-9;]*[mK]/', '', rtrim($raw));
|
|
if (!$clean) continue;
|
|
if (preg_match('/\[(?:ERROR|WARN|CRITICAL|FAILED)\]/i', $clean) ||
|
|
preg_match('/\b(?:ERROR|CRITICAL):\s/i', $clean) ||
|
|
str_contains($clean, '✗') ||
|
|
(str_contains($clean, '⚠') && !str_contains($clean, '♥'))) {
|
|
$lastErr = mb_substr($clean, 0, 220);
|
|
break;
|
|
}
|
|
}
|
|
// Fall back to last non-blank line if no error pattern found in log
|
|
if ($lastErr === null) {
|
|
foreach (array_reverse($lines) as $raw) {
|
|
$clean = preg_replace('/\033\[[0-9;]*[mK]/', '', rtrim($raw));
|
|
if ($clean && !str_starts_with($clean, '──')) { $lastErr = mb_substr($clean, 0, 220); break; }
|
|
}
|
|
}
|
|
if ($lastErr !== null)
|
|
$errors[] = ['script' => $script, 'line' => $lastErr, 'ts' => (int)$lf->getMTime()];
|
|
}
|
|
} catch (Exception $e) {}
|
|
usort($errors, fn($a, $b) => $b['ts'] - $a['ts']);
|
|
}
|
|
$out['errors'] = array_slice($errors, 0, 20);
|
|
|
|
// ── Partner reachability ───────────────────────────────────────────────────
|
|
$cacheFile = '/tmp/vv_partner_cache.json';
|
|
$cacheTtl = 30;
|
|
$partnerData = null;
|
|
|
|
if (file_exists($cacheFile) && (time() - (int)filemtime($cacheFile)) < $cacheTtl) {
|
|
$partnerData = json_decode(file_get_contents($cacheFile), true);
|
|
} else {
|
|
// Discover partner hostname dynamically from master.conf (works for any number of hosts)
|
|
$vars = vv_conf_vars();
|
|
$mine = vv_get_hostname();
|
|
$partnerHost = null;
|
|
foreach ($vars as $k => $v) {
|
|
if (preg_match('/^HOST\d+$/', $k) && $v !== '' && strcasecmp($v, $mine) !== 0) {
|
|
$partnerHost = $v; // first non-self host is the partner
|
|
break;
|
|
}
|
|
}
|
|
|
|
if ($partnerHost) {
|
|
// Resolve via Tailscale — mirrors common.sh resolve_tailscale_ip().
|
|
// Uses `tailscale ip -4` so it survives IP changes; never relies on local DNS.
|
|
$partnerIp = vv_resolve_tailscale_ip($partnerHost);
|
|
$target = $partnerIp ?: $partnerHost;
|
|
$start = microtime(true);
|
|
$result = shell_exec('ping -c1 -W2 ' . escapeshellarg($target) . ' 2>&1');
|
|
$elapsed = (int)round((microtime(true) - $start) * 1000);
|
|
$reached = str_contains((string)$result, '1 received')
|
|
|| str_contains((string)$result, '1 packets received');
|
|
$partnerData = [
|
|
'host' => $partnerHost,
|
|
'reachable' => $reached,
|
|
'latency' => $reached ? $elapsed : null,
|
|
];
|
|
@file_put_contents($cacheFile, json_encode($partnerData));
|
|
}
|
|
}
|
|
$out['partner'] = $partnerData;
|
|
|
|
echo json_encode($out);
|