296 lines
12 KiB
PHP
296 lines
12 KiB
PHP
<?php
|
|
require_once __DIR__ . '/common.php';
|
|
|
|
// Monitor-page-specific helpers — partner state, fallback state, watchdog summary, scripts status.
|
|
|
|
function vv_partner_state(): array {
|
|
$vars = vv_conf_vars();
|
|
$myName = trim(shell_exec('hostname -s') ?: '');
|
|
|
|
// Parse Tailscale peer status once
|
|
$tsData = json_decode(shell_exec('tailscale status --json 2>/dev/null') ?: '{}', true) ?? [];
|
|
$tsPeers = [];
|
|
foreach ($tsData['Peer'] ?? [] as $peer) {
|
|
// DNSName is "hostname.tailnet.ts.net." — take the first label (full, not truncated)
|
|
$dns = $peer['DNSName'] ?? '';
|
|
$h = $dns ? strtolower(explode('.', $dns)[0]) : strtolower($peer['HostName'] ?? '');
|
|
if ($h) $tsPeers[$h] = (bool)($peer['Online'] ?? false);
|
|
}
|
|
|
|
$hostIds = array_filter(array_keys($vars), fn($k) => preg_match('/^HOST\d+$/', $k) && ($vars[$k] ?? '') !== '');
|
|
sort($hostIds);
|
|
|
|
$hosts = [];
|
|
foreach ($hostIds as $id) {
|
|
$hostname = $vars[$id] ?? '';
|
|
if (!$hostname) continue;
|
|
$isMe = strcasecmp($hostname, $myName) === 0;
|
|
$isOwner = strcasecmp($id, $vars['PARTNERSHIP_OWNER_HOST'] ?? '') === 0;
|
|
$online = $isMe ? true : ($tsPeers[strtolower($hostname)] ?? null);
|
|
$hosts[] = [
|
|
'id' => $id,
|
|
'hostname' => $hostname,
|
|
'owner' => $vars[$id . '_OWNER'] ?? '',
|
|
'is_me' => $isMe,
|
|
'is_owner' => $isOwner,
|
|
'online' => $online,
|
|
];
|
|
}
|
|
|
|
return [
|
|
'enabled' => ($vars['PARTNERSHIP_ENABLED'] ?? 'false') === 'true',
|
|
'owner_host' => $vars['PARTNERSHIP_OWNER_HOST'] ?? '',
|
|
'sync_min' => (int)($vars['PARTNERSHIP_SYNC_INTERVAL'] ?? 15),
|
|
'hosts' => $hosts,
|
|
];
|
|
}
|
|
|
|
function vv_fallback_state(): array {
|
|
// State file written by fallback.sh
|
|
$stateFile = '/tmp/fallback_state.db';
|
|
if (!file_exists($stateFile)) return ['state' => 'UNKNOWN'];
|
|
$raw = [];
|
|
foreach (file($stateFile) ?: [] as $line) {
|
|
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
|
$raw[trim($k)] = trim($v);
|
|
}
|
|
return [
|
|
'state' => $raw['state'] ?? 'UNKNOWN',
|
|
'failover_start' => $raw['failover_start'] ?? '0',
|
|
'tier2_started' => $raw['tier2_started'] ?? 'false',
|
|
'tier3_started' => $raw['tier3_started'] ?? 'false',
|
|
'tier4_started' => $raw['tier4_started'] ?? 'false',
|
|
];
|
|
}
|
|
|
|
function vv_fallback_active(): array {
|
|
$vars = vv_conf_vars();
|
|
$myName = trim(shell_exec('hostname -s') ?: '');
|
|
|
|
// Identify which HOST id we are
|
|
$allHostIds = array_filter(array_keys($vars), fn($k) => preg_match('/^HOST\d+$/', $k) && ($vars[$k] ?? '') !== '');
|
|
sort($allHostIds);
|
|
$myId = null;
|
|
foreach ($allHostIds as $id) {
|
|
if (strcasecmp($vars[$id] ?? '', $myName) === 0) { $myId = $id; break; }
|
|
}
|
|
if (!$myId) return [];
|
|
|
|
// Running containers: name → image
|
|
$running = [];
|
|
$psOut = shell_exec("docker ps --format '{\"n\":\"{{.Names}}\",\"i\":\"{{.Image}}\"}' 2>/dev/null") ?: '';
|
|
foreach (explode("\n", trim($psOut)) as $line) {
|
|
$c = json_decode($line, true);
|
|
if ($c) $running[strtolower($c['n'])] = $c['i'];
|
|
}
|
|
|
|
// Parse FALLBACK arrays from this host's conf
|
|
$confFile = strtolower($myId) . '.conf';
|
|
$rawConf = vv_read_conf_raw($confFile);
|
|
|
|
$result = [];
|
|
foreach ($allHostIds as $covered) {
|
|
if ($covered === $myId) continue;
|
|
$coveredHostname = $vars[$covered] ?? '';
|
|
if (!$coveredHostname) continue;
|
|
|
|
$names = [];
|
|
for ($tier = 1; $tier <= 4; $tier++)
|
|
$names = array_merge($names, vv_parse_bash_array($rawConf, "FALLBACK_{$myId}_COVERS_{$covered}_TIER{$tier}"));
|
|
|
|
$active = [];
|
|
foreach ($names as $name) {
|
|
if (isset($running[strtolower($name)]))
|
|
$active[] = ['name' => $name, 'image' => $running[strtolower($name)]];
|
|
}
|
|
|
|
if ($active) $result[] = ['host_id' => $covered, 'hostname' => $coveredHostname, 'containers' => $active];
|
|
}
|
|
|
|
return $result;
|
|
}
|
|
|
|
function vv_watchdog_summary(): array {
|
|
$parseKv = function(string $raw): array {
|
|
$out = [];
|
|
foreach (explode("\n", $raw) as $line) {
|
|
$line = trim($line);
|
|
if (str_contains($line, ':')) { [$k, $v] = explode(':', $line, 2); $out[trim($k)] = trim($v); }
|
|
elseif (str_contains($line, '=')) { [$k, $v] = explode('=', $line, 2); $out[trim($k)] = trim($v, '"\''); }
|
|
}
|
|
return $out;
|
|
};
|
|
|
|
$dock = $parseKv(@file_get_contents('/tmp/container_watchdog_state.db') ?: '');
|
|
$rw = $parseKv(@file_get_contents('/tmp/resource_watchdog_state.db') ?: '');
|
|
|
|
$ctrStrikes = [];
|
|
foreach ($dock as $k => $v) {
|
|
if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0)
|
|
$ctrStrikes[$k] = (int)$v;
|
|
}
|
|
|
|
// Recent restarts (24 h)
|
|
$restartLog = '/mnt/user/appdata/Varaverk/data/container_restart_history.db';
|
|
$restartRaw = @file_get_contents($restartLog) ?: '';
|
|
$cutoff = time() - 86400;
|
|
$restarts = [];
|
|
foreach (explode("\n", trim($restartRaw)) as $line) {
|
|
if (!$line || !str_contains($line, '|')) continue;
|
|
[$name, $ts] = explode('|', $line, 2);
|
|
if ((int)$ts >= $cutoff) $restarts[] = ['name' => trim($name), 'ts' => (int)$ts];
|
|
}
|
|
usort($restarts, fn($a, $b) => $b['ts'] - $a['ts']);
|
|
|
|
// Reboots (12 h)
|
|
$rebootRaw = @file_get_contents('/boot/config/system_watchdog_reboots.db') ?: '';
|
|
$rbootCutoff = time() - 43200;
|
|
$reboots = 0;
|
|
foreach (explode("\n", trim($rebootRaw)) as $line) {
|
|
if ((int)trim($line) >= $rbootCutoff) $reboots++;
|
|
}
|
|
|
|
$rwLevel = (int)($rw['rm_action_level'] ?? 0);
|
|
$daemonStrikes = (int)($dock['daemon_strikes'] ?? 0);
|
|
$oomCount = (int)trim(@file_get_contents('/tmp/system_watchdog_oom.db') ?: '0');
|
|
|
|
// ── Stability watchdog strikes (/tmp/system_watchdog_state.db) ───────────
|
|
$stabRaw = @file_get_contents('/tmp/system_watchdog_state.db') ?: '';
|
|
$stabStrikes = [];
|
|
foreach (explode("\n", $stabRaw) as $line) {
|
|
$line = trim($line);
|
|
if (!$line || !str_contains($line, ':')) continue;
|
|
[$k, $v] = explode(':', $line, 2);
|
|
$count = (int)trim($v);
|
|
if ($count > 0) $stabStrikes[trim($k)] = $count;
|
|
}
|
|
|
|
// ── Storage watchdog strikes (/tmp/storage_watchdog_state.db) ────────────
|
|
$storRaw = @file_get_contents('/tmp/storage_watchdog_state.db') ?: '';
|
|
$growthStrikes = []; $logStrikes = [];
|
|
foreach (explode("\n", $storRaw) as $line) {
|
|
$line = trim($line);
|
|
if (!$line || !str_contains($line, ':')) continue;
|
|
[$k, $v] = explode(':', $line, 2);
|
|
$count = (int)trim($v);
|
|
if ($count <= 0) continue;
|
|
$key = trim($k);
|
|
if (str_starts_with($key, 'appdata_growth_'))
|
|
$growthStrikes[substr($key, strlen('appdata_growth_'))] = $count;
|
|
elseif (str_starts_with($key, 'appdata_log_'))
|
|
$logStrikes[substr($key, strlen('appdata_log_'))] = $count;
|
|
}
|
|
|
|
// ── Network watchdog NPM strikes (/tmp/network_watchdog_state.db) ────────
|
|
$netRaw = @file_get_contents('/tmp/network_watchdog_state.db') ?: '';
|
|
$npmStrikes = 0;
|
|
foreach (explode("\n", $netRaw) as $line) {
|
|
$line = trim($line);
|
|
if (str_starts_with($line, 'npm:')) $npmStrikes = (int)trim(substr($line, 4));
|
|
}
|
|
|
|
// ── Stability live stats ──────────────────────────────────────────────────
|
|
$dfPct = function(string $path): int {
|
|
$out = shell_exec("df " . escapeshellarg($path) . " --output=pcent 2>/dev/null | tail -1") ?: '';
|
|
return (int)trim(str_replace('%', '', $out));
|
|
};
|
|
|
|
$memRaw = @file_get_contents('/proc/meminfo') ?: '';
|
|
$memAvail = 0;
|
|
if (preg_match('/^MemAvailable:\s+(\d+)/m', $memRaw, $m)) $memAvail = (int)$m[1];
|
|
|
|
$loadRaw = @file_get_contents('/proc/loadavg') ?: '0';
|
|
$load1 = (float)explode(' ', trim($loadRaw))[0];
|
|
|
|
$cpuTemp = null;
|
|
$sensorsOut = shell_exec("sensors 2>/dev/null | grep -E 'Core 0|Package id 0|Tdie|Tctl|CPU Temp' | grep -oE '[0-9]+\\.[0-9]+' | sort -n | tail -1") ?: '';
|
|
if ($sensorsOut && is_numeric(trim($sensorsOut))) $cpuTemp = (int)round((float)trim($sensorsOut));
|
|
|
|
$zombies = (int)trim(shell_exec("ps -eo stat 2>/dev/null | grep -c '^Z'") ?: '0');
|
|
|
|
$nic = trim(shell_exec("ip route show default 2>/dev/null | awk 'NR==1{print \$5}'") ?: 'eth0') ?: 'eth0';
|
|
$nicState = trim(@file_get_contents("/sys/class/net/$nic/operstate") ?: 'unknown');
|
|
$sshdOk = (int)trim(shell_exec('pgrep -c sshd 2>/dev/null') ?: '0') > 0;
|
|
|
|
$healthy = empty($ctrStrikes) && empty($stabStrikes) && empty($growthStrikes) && empty($logStrikes)
|
|
&& $rwLevel === 0 && $daemonStrikes === 0 && $oomCount === 0 && $reboots === 0
|
|
&& $npmStrikes === 0 && $nicState === 'up' && $sshdOk;
|
|
|
|
return [
|
|
'healthy' => $healthy,
|
|
'ctr_strikes' => $ctrStrikes,
|
|
'rw_level' => $rwLevel,
|
|
'rw_paused' => array_values(array_filter(explode(',', $rw['rm_paused_containers'] ?? ''))),
|
|
'rw_stopped' => array_values(array_filter(explode(',', $rw['rm_stopped_containers'] ?? ''))),
|
|
'daemon_strikes' => $daemonStrikes,
|
|
'oom_count' => $oomCount,
|
|
'reboots_12h' => $reboots,
|
|
'restarts_24h' => array_slice($restarts, 0, 6),
|
|
'restart_count' => count($restarts),
|
|
'stability' => [
|
|
'rootfs_pct' => $dfPct('/'),
|
|
'log_pct' => $dfPct('/var/log'),
|
|
'tmp_pct' => $dfPct('/tmp'),
|
|
'ram_free_gb' => round($memAvail / 1048576, 1),
|
|
'load_1min' => $load1,
|
|
'cpu_temp' => $cpuTemp,
|
|
'zombies' => $zombies,
|
|
'nic' => $nic,
|
|
'nic_state' => $nicState,
|
|
'sshd_ok' => $sshdOk,
|
|
'strikes' => $stabStrikes,
|
|
],
|
|
'storage_wd' => [
|
|
'growth_strikes' => $growthStrikes,
|
|
'log_strikes' => $logStrikes,
|
|
],
|
|
'network_wd' => [
|
|
'npm_strikes' => $npmStrikes,
|
|
],
|
|
];
|
|
}
|
|
|
|
function vv_scripts_status(): array {
|
|
$logDir = LOG_DIR;
|
|
$statFiles = array_merge(
|
|
glob("$logDir/*.json") ?: [],
|
|
glob("$logDir/*/*.json") ?: []
|
|
);
|
|
|
|
$scripts = [];
|
|
foreach ($statFiles as $statFile) {
|
|
$stat = json_decode(@file_get_contents($statFile) ?: '{}', true) ?: [];
|
|
$status = $stat['status'] ?? 'unknown';
|
|
|
|
// Stale running — PID gone (crash or reboot with no cleanup)
|
|
if ($status === 'running' && !empty($stat['pid'])) {
|
|
if (!file_exists("/proc/{$stat['pid']}")) $status = 'error';
|
|
}
|
|
|
|
$id = $stat['id'] ?? basename($statFile, '.json');
|
|
$name = basename(preg_replace('/\.sh$/', '', $id));
|
|
$ts = (int)($stat['end'] ?? $stat['start'] ?? @filemtime($statFile) ?: 0);
|
|
|
|
$scripts[] = [
|
|
'name' => $name,
|
|
'last_ts' => $ts,
|
|
'status' => $status,
|
|
'running' => $status === 'running',
|
|
'exit' => $stat['exit'] ?? null,
|
|
'duration' => isset($stat['start'], $stat['end'])
|
|
? (int)$stat['end'] - (int)$stat['start'] : null,
|
|
];
|
|
}
|
|
|
|
usort($scripts, fn($a, $b) => ($b['last_ts'] ?? 0) <=> ($a['last_ts'] ?? 0));
|
|
$scripts = array_slice($scripts, 0, 12);
|
|
|
|
return [
|
|
'scripts' => $scripts,
|
|
'running_count' => count(array_filter($scripts, fn($s) => $s['status'] === 'running')),
|
|
'ok_count' => count(array_filter($scripts, fn($s) => $s['status'] === 'ok')),
|
|
'warn_count' => count(array_filter($scripts, fn($s) => $s['status'] === 'warn')),
|
|
'error_count' => count(array_filter($scripts, fn($s) => $s['status'] === 'error')),
|
|
];
|
|
}
|