541 lines
24 KiB
PHP
541 lines
24 KiB
PHP
<?php
|
|
require_once __DIR__ . '/common.php';
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
// PURPOSE
|
|
// Monitor-page roll-ups that are not raw system metrics: partner reachability, fallback
|
|
// state, the single-glance watchdog health summary, script run status, and rsync progress.
|
|
// Raw hardware numbers come from common.php; this file answers "is anything wrong".
|
|
//
|
|
// DESIGN PRINCIPLES
|
|
// One boolean has to be trustworthy.
|
|
// vv_watchdog_summary() reduces every watchdog's state to 'healthy'. It is the only
|
|
// thing most people look at, so it is conjunctive — healthy requires every strike set
|
|
// empty, every level zero, the NIC up and sshd alive. Any doubt resolves to not-healthy.
|
|
//
|
|
// State paths derive from STATE_DIR, never hardcoded.
|
|
// The watchdogs write under STATE_DIR, which follows SCRIPTS_DIR through a storage-mode
|
|
// migration. Hardcoding an absolute path here silently decouples the page from the
|
|
// scripts — see OPERATIONAL SAFEGUARDS.
|
|
//
|
|
// Summarises; does not re-derive.
|
|
// Strike counts come from the files the watchdogs wrote. This file never recomputes
|
|
// whether a container is unhealthy — that decision belongs to the watchdog that owns it.
|
|
//
|
|
// OPERATIONAL SAFEGUARDS
|
|
// Missing state must not read as healthy — and once did.
|
|
// Six state files were read from /tmp while the watchdogs write to STATE_DIR. Every
|
|
// read returned empty, every strike set came back clear, and 'healthy' was therefore
|
|
// always true: a permanent false all-clear on the page whose whole job is raising the
|
|
// alarm. Fixed 2026-08-02. If a strike set ever looks suspiciously empty, verify the
|
|
// path against where the watchdog actually writes before trusting it.
|
|
//
|
|
// Live stability probes are cheap and time-boxed.
|
|
// df, sensors, pgrep and ps run per render, so each is a single command with stderr
|
|
// discarded and a scalar result. Nothing here iterates over containers or disks.
|
|
//
|
|
// Shell arguments are escaped.
|
|
// Paths passed to df go through escapeshellarg(); the NIC name is read from sysfs
|
|
// rather than interpolated from user input.
|
|
//
|
|
// Read-only. Reports on watchdogs, fallback and scripts; never starts, stops, or clears any
|
|
// of them.
|
|
//
|
|
// EXPORTS
|
|
// vv_partner_state() partner reachability and identity
|
|
// vv_fallback_state() current fallback state for this host
|
|
// vv_fallback_active() whether this host is currently covering, and what
|
|
// vv_watchdog_summary() the health roll-up described above
|
|
// vv_scripts_status() last-run status per scheduled script
|
|
// vv_rsync_status() current/last rsync progress
|
|
//
|
|
// CONFIGURATION
|
|
// STATE_DIR fallback_state.db, container/resource/system/storage/network watchdog state,
|
|
// system_watchdog_oom.db, system_watchdog_reboots.db
|
|
// DATA_DIR container_restart_history.db
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
function vv_partner_state(): array {
|
|
$vars = vv_conf_vars();
|
|
$myHostId = strtoupper(vv_detect_host());
|
|
|
|
// Parse Tailscale peer status once
|
|
$tsData = json_decode(shell_exec('tailscale status --json 2>/dev/null') ?: '{}', true) ?? [];
|
|
$tsPeers = [];
|
|
foreach ($tsData['Peer'] ?? [] as $peer) {
|
|
// DNSName is "hostname.tailnet.ts.net." — take the first label (full, not truncated)
|
|
$dns = $peer['DNSName'] ?? '';
|
|
$h = $dns ? strtolower(explode('.', $dns)[0]) : strtolower($peer['HostName'] ?? '');
|
|
if ($h) $tsPeers[$h] = (bool)($peer['Online'] ?? false);
|
|
}
|
|
|
|
$hostIds = array_filter(array_keys($vars), fn($k) => preg_match('/^HOST\d+$/', $k) && ($vars[$k] ?? '') !== '');
|
|
sort($hostIds);
|
|
|
|
$setupDb = vv_setup_state_read();
|
|
|
|
// HOST<n>_PHASE*_DONE always describes the MIRROR — it is the owner's record of how far it got
|
|
// provisioning the other side, so the owner's own slot has no such flag and never will. Read from
|
|
// the mirror that made the owner compute phase 0 and render "Not provisioned" on the very host
|
|
// that had just finished onboarding it. An ACTIVE partnership is the authority on whether the
|
|
// mesh is provisioned; the flags only say who did what to whom. Same rule as vv_pt_nodes().
|
|
$ptFile = STATE_DIR . '/partnership_' . vv_get_hostname() . '.db';
|
|
$ptRaw = [];
|
|
foreach ((is_readable($ptFile) ? file($ptFile) : []) ?: [] as $line) {
|
|
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
|
$ptRaw[trim($k)] = trim($v, "\"'");
|
|
}
|
|
$ptActive = ($ptRaw['state'] ?? '') === 'ACTIVE';
|
|
$ptMembers = array_filter([strtolower($ptRaw['owner'] ?? ''), strtolower($ptRaw['mirror'] ?? '')]);
|
|
|
|
$hosts = [];
|
|
foreach ($hostIds as $id) {
|
|
$hostname = $vars[$id] ?? '';
|
|
if (!$hostname) continue;
|
|
$isMe = ($id === $myHostId);
|
|
$isOwner = strcasecmp($id, $vars['PARTNERSHIP_OWNER_HOST'] ?? '') === 0;
|
|
$online = $isMe ? true : ($tsPeers[strtolower($hostname)] ?? null);
|
|
$onboardPhase = $isMe ? null
|
|
: (($setupDb[$id . '_PHASE2_DONE'] ?? '') === 'true' ? 2
|
|
: (($setupDb[$id . '_PHASE1_DONE'] ?? '') === 'true' ? 1 : 0));
|
|
if ($onboardPhase !== null && $onboardPhase < 2 && $ptActive
|
|
&& in_array(strtolower($hostname), $ptMembers, true)) {
|
|
$onboardPhase = 2;
|
|
}
|
|
$hosts[] = [
|
|
'id' => $id,
|
|
'hostname' => $hostname,
|
|
'owner' => $vars[$id . '_OWNER'] ?? '',
|
|
'is_me' => $isMe,
|
|
'is_owner' => $isOwner,
|
|
'online' => $online,
|
|
'onboard_phase' => $onboardPhase,
|
|
];
|
|
}
|
|
|
|
return [
|
|
'enabled' => ($vars['PARTNERSHIP_ENABLED'] ?? 'false') === 'true',
|
|
'owner_host' => $vars['PARTNERSHIP_OWNER_HOST'] ?? '',
|
|
'sync_min' => (int)($vars['PARTNERSHIP_SYNC_INTERVAL'] ?? 15),
|
|
'hosts' => $hosts,
|
|
];
|
|
}
|
|
|
|
function vv_fallback_state(): array {
|
|
$vars = vv_conf_vars();
|
|
$enabled = ($vars['FALLBACK_ENABLED'] ?? 'false') === 'true';
|
|
$interval = (int)($vars['FALLBACK_CHECK_INTERVAL'] ?? 30);
|
|
$reqStrikes = (int)($vars['FALLBACK_HANDBACK_STRIKES'] ?? 3);
|
|
$suspendAfter = (int)($vars['FALLBACK_PARTNERSHIP_SUSPEND_AFTER'] ?? 120);
|
|
|
|
$stateFile = STATE_DIR . '/fallback_state.db';
|
|
if (!file_exists($stateFile)) {
|
|
return ['state' => 'UNKNOWN', 'enabled' => $enabled, 'check_interval' => $interval,
|
|
'handback_strikes' => 0, 'handback_strikes_required' => $reqStrikes,
|
|
'partnership_suspended' => false,
|
|
'partner_lost_at' => 0, 'partnership_suspend_after' => $suspendAfter];
|
|
}
|
|
$raw = [];
|
|
foreach (file($stateFile) ?: [] as $line) {
|
|
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
|
$raw[trim($k)] = trim($v);
|
|
}
|
|
return [
|
|
'state' => $raw['state'] ?? 'UNKNOWN',
|
|
'failover_start' => $raw['failover_start'] ?? '0',
|
|
'tier2_started' => $raw['tier2_started'] ?? 'false',
|
|
'tier3_started' => $raw['tier3_started'] ?? 'false',
|
|
'tier4_started' => $raw['tier4_started'] ?? 'false',
|
|
'handback_strikes' => (int)($raw['handback_strikes'] ?? 0),
|
|
'partnership_suspended' => ($raw['partnership_suspended'] ?? 'false') === 'true',
|
|
'partner_lost_at' => (int)($raw['partner_lost_at'] ?? 0),
|
|
'enabled' => $enabled,
|
|
'check_interval' => $interval,
|
|
'handback_strikes_required' => $reqStrikes,
|
|
'partnership_suspend_after' => $suspendAfter,
|
|
];
|
|
}
|
|
|
|
function vv_fallback_active(): array {
|
|
$vars = vv_conf_vars();
|
|
|
|
// Identify which HOST id we are
|
|
$allHostIds = array_filter(array_keys($vars), fn($k) => preg_match('/^HOST\d+$/', $k) && ($vars[$k] ?? '') !== '');
|
|
sort($allHostIds);
|
|
$myId = strtoupper(vv_detect_host());
|
|
if ($myId === 'UNKNOWN' || !in_array($myId, $allHostIds, true)) return [];
|
|
|
|
// Running containers: name → image
|
|
$running = [];
|
|
$psOut = shell_exec("docker ps --format '{\"n\":\"{{.Names}}\",\"i\":\"{{.Image}}\"}' 2>/dev/null") ?: '';
|
|
foreach (explode("\n", trim($psOut)) as $line) {
|
|
$c = json_decode($line, true);
|
|
if ($c) $running[strtolower($c['n'])] = $c['i'];
|
|
}
|
|
|
|
$result = [];
|
|
foreach ($allHostIds as $covered) {
|
|
if ($covered === $myId) continue;
|
|
$coveredHostname = $vars[$covered] ?? '';
|
|
if (!$coveredHostname) continue;
|
|
|
|
// Covered host defines its own recovery profile — read from their conf
|
|
$coveredRaw = vv_read_conf_raw(strtolower($covered) . '.conf');
|
|
$names = [];
|
|
for ($tier = 1; $tier <= 4; $tier++)
|
|
$names = array_merge($names, vv_parse_bash_array($coveredRaw, "FALLBACK_{$covered}_TIER{$tier}"));
|
|
|
|
$active = [];
|
|
foreach ($names as $name) {
|
|
if (isset($running[strtolower($name)]))
|
|
$active[] = ['name' => $name, 'image' => $running[strtolower($name)]];
|
|
}
|
|
|
|
if ($active) $result[] = ['host_id' => $covered, 'hostname' => $coveredHostname, 'containers' => $active];
|
|
}
|
|
|
|
return $result;
|
|
}
|
|
|
|
function vv_watchdog_summary(): array {
|
|
$parseKv = function(string $raw): array {
|
|
$out = [];
|
|
foreach (explode("\n", $raw) as $line) {
|
|
$line = trim($line);
|
|
if (str_contains($line, ':')) { [$k, $v] = explode(':', $line, 2); $out[trim($k)] = trim($v); }
|
|
elseif (str_contains($line, '=')) { [$k, $v] = explode('=', $line, 2); $out[trim($k)] = trim($v, '"\''); }
|
|
}
|
|
return $out;
|
|
};
|
|
|
|
$dock = $parseKv(@file_get_contents(STATE_DIR . '/container_watchdog_state.db') ?: '');
|
|
$rw = $parseKv(@file_get_contents(STATE_DIR . '/resource_watchdog_state.db') ?: '');
|
|
|
|
$ctrStrikes = [];
|
|
foreach ($dock as $k => $v) {
|
|
if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0)
|
|
$ctrStrikes[$k] = (int)$v;
|
|
}
|
|
|
|
// Recent restarts (24 h)
|
|
$restartLog = DB_DIR . '/container_restart_history.db';
|
|
$restartRaw = @file_get_contents($restartLog) ?: '';
|
|
$cutoff = time() - 86400;
|
|
$restarts = [];
|
|
foreach (explode("\n", trim($restartRaw)) as $line) {
|
|
if (!$line || !str_contains($line, '|')) continue;
|
|
// The second field is a formatted local date, not an epoch — see
|
|
// vv_wd_restart_ts() in watchdog.php for why, and for what casting it with (int)
|
|
// silently did to this list for as long as it has existed.
|
|
[$name, $raw] = explode('|', $line, 2);
|
|
$ts = vv_wd_restart_ts(trim($raw));
|
|
if ($ts >= $cutoff) $restarts[] = ['name' => trim($name), 'ts' => $ts];
|
|
}
|
|
usort($restarts, fn($a, $b) => $b['ts'] - $a['ts']);
|
|
|
|
// Reboots (12 h)
|
|
$rebootRaw = @file_get_contents(STATE_DIR . '/system_watchdog_reboots.db') ?: '';
|
|
$rbootCutoff = time() - 43200;
|
|
$reboots = 0;
|
|
foreach (explode("\n", trim($rebootRaw)) as $line) {
|
|
if ((int)trim($line) >= $rbootCutoff) $reboots++;
|
|
}
|
|
|
|
$rwLevel = (int)($rw['rm_action_level'] ?? 0);
|
|
$daemonStrikes = (int)($dock['daemon_strikes'] ?? 0);
|
|
$oomCount = (int)trim(@file_get_contents(STATE_DIR . '/system_watchdog_oom.db') ?: '0');
|
|
|
|
// ── Stability watchdog strikes (STATE_DIR) ───────────
|
|
$stabRaw = @file_get_contents(STATE_DIR . '/system_watchdog_state.db') ?: '';
|
|
$stabStrikes = [];
|
|
foreach (explode("\n", $stabRaw) as $line) {
|
|
$line = trim($line);
|
|
if (!$line || !str_contains($line, ':')) continue;
|
|
[$k, $v] = explode(':', $line, 2);
|
|
$count = (int)trim($v);
|
|
if ($count > 0) $stabStrikes[trim($k)] = $count;
|
|
}
|
|
|
|
// ── Storage watchdog strikes (STATE_DIR) ────────────
|
|
$storRaw = @file_get_contents(STATE_DIR . '/storage_watchdog_state.db') ?: '';
|
|
$growthStrikes = []; $logStrikes = [];
|
|
foreach (explode("\n", $storRaw) as $line) {
|
|
$line = trim($line);
|
|
if (!$line || !str_contains($line, ':')) continue;
|
|
[$k, $v] = explode(':', $line, 2);
|
|
$count = (int)trim($v);
|
|
if ($count <= 0) continue;
|
|
$key = trim($k);
|
|
if (str_starts_with($key, 'appdata_growth_'))
|
|
$growthStrikes[substr($key, strlen('appdata_growth_'))] = $count;
|
|
elseif (str_starts_with($key, 'appdata_log_'))
|
|
$logStrikes[substr($key, strlen('appdata_log_'))] = $count;
|
|
}
|
|
|
|
// ── Network watchdog NPM strikes (STATE_DIR) ────────
|
|
$netRaw = @file_get_contents(STATE_DIR . '/network_watchdog_state.db') ?: '';
|
|
$npmStrikes = 0;
|
|
foreach (explode("\n", $netRaw) as $line) {
|
|
$line = trim($line);
|
|
if (str_starts_with($line, 'npm:')) $npmStrikes = (int)trim(substr($line, 4));
|
|
}
|
|
|
|
// ── Stability live stats ──────────────────────────────────────────────────
|
|
$dfPct = function(string $path): int {
|
|
$out = shell_exec("df " . escapeshellarg($path) . " --output=pcent 2>/dev/null | tail -1") ?: '';
|
|
return (int)trim(str_replace('%', '', $out));
|
|
};
|
|
|
|
$memRaw = @file_get_contents('/proc/meminfo') ?: '';
|
|
$memAvail = 0;
|
|
if (preg_match('/^MemAvailable:\s+(\d+)/m', $memRaw, $m)) $memAvail = (int)$m[1];
|
|
|
|
$loadRaw = @file_get_contents('/proc/loadavg') ?: '0';
|
|
$load1 = (float)explode(' ', trim($loadRaw))[0];
|
|
|
|
$cpuTemp = vv_cpu_temp();
|
|
|
|
$zombies = (int)trim(shell_exec("ps -eo stat 2>/dev/null | grep -c '^Z'") ?: '0');
|
|
|
|
$fileNr = explode("\t", trim(@file_get_contents('/proc/sys/fs/file-nr') ?: '0 0 1'));
|
|
$fdOpen = max(0, (int)($fileNr[0] ?? 0) - (int)($fileNr[1] ?? 0));
|
|
$fdMax = max(1, (int)($fileNr[2] ?? 1));
|
|
$fdPct = round($fdOpen / $fdMax * 100, 1);
|
|
|
|
$nic = trim(shell_exec("ip route show default 2>/dev/null | awk 'NR==1{print \$5}'") ?: 'eth0') ?: 'eth0';
|
|
$nicState = trim(@file_get_contents("/sys/class/net/$nic/operstate") ?: 'unknown');
|
|
$sshdOk = (int)trim(shell_exec('pgrep -c sshd 2>/dev/null') ?: '0') > 0;
|
|
|
|
$healthy = empty($ctrStrikes) && empty($stabStrikes) && empty($growthStrikes) && empty($logStrikes)
|
|
&& $rwLevel === 0 && $daemonStrikes === 0 && $oomCount === 0 && $reboots === 0
|
|
&& $npmStrikes === 0 && $nicState === 'up' && $sshdOk;
|
|
|
|
return [
|
|
'healthy' => $healthy,
|
|
'ctr_strikes' => $ctrStrikes,
|
|
'rw_level' => $rwLevel,
|
|
'rw_paused' => array_values(array_filter(explode(',', $rw['rm_paused_containers'] ?? ''))),
|
|
'rw_stopped' => array_values(array_filter(explode(',', $rw['rm_stopped_containers'] ?? ''))),
|
|
'daemon_strikes' => $daemonStrikes,
|
|
'oom_count' => $oomCount,
|
|
'reboots_12h' => $reboots,
|
|
'restarts_24h' => array_slice($restarts, 0, 6),
|
|
'restart_count' => count($restarts),
|
|
'stability' => [
|
|
'rootfs_pct' => $dfPct('/'),
|
|
'log_pct' => $dfPct('/var/log'),
|
|
'tmp_pct' => $dfPct('/tmp'),
|
|
'ram_free_gb' => round($memAvail / 1048576, 1),
|
|
'load_1min' => $load1,
|
|
'cpu_temp' => $cpuTemp,
|
|
'zombies' => $zombies,
|
|
'fd_open' => $fdOpen,
|
|
'fd_max' => $fdMax,
|
|
'fd_pct' => $fdPct,
|
|
'nic' => $nic,
|
|
'nic_state' => $nicState,
|
|
'sshd_ok' => $sshdOk,
|
|
'strikes' => $stabStrikes,
|
|
],
|
|
'storage_wd' => [
|
|
'growth_strikes' => $growthStrikes,
|
|
'log_strikes' => $logStrikes,
|
|
],
|
|
'network_wd' => [
|
|
'npm_strikes' => $npmStrikes,
|
|
],
|
|
];
|
|
}
|
|
|
|
// What Varaverk itself is occupying and doing, as opposed to what the machine is.
|
|
//
|
|
// The System card described the host — model, uptime, load — and said nothing about the thing
|
|
// whose dashboard it is. These are the figures that are Varaverk's own and that nothing else on
|
|
// the page reports.
|
|
//
|
|
// The cache size is the one worth watching. VV_CACHE_ROOT is /tmp/varaverk, and on Unraid /tmp is
|
|
// on rootfs, which is RAM — so this directory is memory, not disk, and the arr payload cache is
|
|
// most of it. master.conf says never to move these onto flash, which makes the size the thing to
|
|
// keep an eye on instead. Reported next to the rootfs percentage it consumes, because 187 MB
|
|
// means nothing without the ceiling it counts against.
|
|
//
|
|
// du rather than a recursive PHP walk: both roots are small and page-cached, measured at 3ms
|
|
// each, and this is assembled once a minute by the cache writer rather than per page load.
|
|
function vv_varaverk_state(): array {
|
|
$du = function (string $path): ?int {
|
|
if (!is_dir($path)) return null;
|
|
$out = shell_exec('du -sb ' . escapeshellarg($path) . ' 2>/dev/null');
|
|
return preg_match('/^(\d+)/', (string)$out, $m) ? (int)$m[1] : null;
|
|
};
|
|
|
|
// Locks whose process is still alive. A lock file alone does not mean a job is running —
|
|
// acquire_lock() clears one whose pid is gone, and three were sitting in LOCK_DIR from jobs
|
|
// that finished days ago. Counting files would have reported four jobs running and one
|
|
// actually was.
|
|
$running = [];
|
|
$stale = 0;
|
|
foreach ((array)@glob('/tmp/unraid_locks/*.lock') as $lock) {
|
|
$content = trim((string)@file_get_contents($lock));
|
|
if ($content === '') { $stale++; continue; }
|
|
[$pid, $name] = array_pad(explode(':', $content, 2), 2, '');
|
|
$pid = (int)$pid;
|
|
if ($pid > 1 && @posix_kill($pid, 0)) {
|
|
$running[] = $name !== '' ? $name : basename($lock, '.lock');
|
|
} else {
|
|
$stale++;
|
|
}
|
|
}
|
|
sort($running);
|
|
|
|
// Newest partner conf in the RAM cache. conf_sync fills it; an age climbing past its schedule
|
|
// means the mesh has stopped talking, which nothing else on this page would show.
|
|
$confAge = null;
|
|
foreach ((array)@glob(VV_CONF_RAM_CACHE_DIR . '/host*.conf') as $c) {
|
|
$m = @filemtime($c);
|
|
if ($m && ($confAge === null || (time() - $m) < $confAge)) $confAge = time() - $m;
|
|
}
|
|
|
|
// Storage mode as a fact about where this install actually is, not as the conf toggle's word
|
|
// for it — the toggle is what someone intended and the path is what happened.
|
|
//
|
|
// "internal", not "flash". /boot is the internal mode in Varaverk's own vocabulary and on this
|
|
// hardware it is a mirrored NVMe pool, not a USB stick; calling it flash on the dashboard
|
|
// would invite a write-wear worry that does not apply here.
|
|
$internal = str_starts_with(SCRIPTS_DIR, '/boot');
|
|
|
|
return [
|
|
'cache_root' => VV_CACHE_ROOT,
|
|
'cache_bytes' => $du(VV_CACHE_ROOT),
|
|
'data_bytes' => $du(DATA_DIR),
|
|
'scripts_dir' => SCRIPTS_DIR,
|
|
'storage' => $internal ? 'internal' : 'appdata',
|
|
'commit' => trim((string)shell_exec(
|
|
'git -C ' . escapeshellarg(SCRIPTS_DIR) . ' rev-parse --short HEAD 2>/dev/null')),
|
|
'jobs_running' => $running,
|
|
'locks_stale' => $stale,
|
|
'conf_age_sec' => $confAge,
|
|
];
|
|
}
|
|
|
|
function vv_scripts_status(): array {
|
|
$logDir = LOG_DIR;
|
|
$statFiles = array_merge(
|
|
glob("$logDir/*.json") ?: [],
|
|
glob("$logDir/*/*.json") ?: []
|
|
);
|
|
|
|
$scripts = [];
|
|
foreach ($statFiles as $statFile) {
|
|
$stat = json_decode(@file_get_contents($statFile) ?: '{}', true) ?: [];
|
|
$status = $stat['status'] ?? 'unknown';
|
|
|
|
// Stale running — PID gone (crash or reboot with no cleanup)
|
|
if ($status === 'running' && !empty($stat['pid'])) {
|
|
if (!file_exists("/proc/{$stat['pid']}")) $status = 'error';
|
|
}
|
|
|
|
$id = $stat['id'] ?? basename($statFile, '.json');
|
|
$name = basename(preg_replace('/\.sh$/', '', $id));
|
|
$ts = (int)($stat['end'] ?? $stat['start'] ?? @filemtime($statFile) ?: 0);
|
|
|
|
$scripts[] = [
|
|
'id' => $id,
|
|
'name' => $name,
|
|
'last_ts' => $ts,
|
|
'status' => $status,
|
|
'running' => $status === 'running',
|
|
'exit' => $stat['exit'] ?? null,
|
|
'duration' => isset($stat['start'], $stat['end'])
|
|
? (int)$stat['end'] - (int)$stat['start'] : null,
|
|
];
|
|
}
|
|
|
|
usort($scripts, fn($a, $b) => ($b['last_ts'] ?? 0) <=> ($a['last_ts'] ?? 0));
|
|
$scripts = array_slice($scripts, 0, 12);
|
|
|
|
return [
|
|
'scripts' => $scripts,
|
|
'running_count' => count(array_filter($scripts, fn($s) => $s['status'] === 'running')),
|
|
'ok_count' => count(array_filter($scripts, fn($s) => $s['status'] === 'ok')),
|
|
'warn_count' => count(array_filter($scripts, fn($s) => $s['status'] === 'warn')),
|
|
'error_count' => count(array_filter($scripts, fn($s) => $s['status'] === 'error')),
|
|
];
|
|
}
|
|
|
|
function vv_rsync_status(): array {
|
|
$vars = vv_conf_vars();
|
|
|
|
$enabled = ($vars['RSYNC_ENABLED'] ?? 'true') !== 'false';
|
|
|
|
$windows = [
|
|
'critical' => ($vars['CRITICAL_RSYNC_ENABLED'] ?? 'false') !== 'false',
|
|
'daily' => ($vars['DAILY_RSYNC_ENABLED'] ?? 'false') !== 'false',
|
|
'intermediate' => ($vars['INTERMEDIATE_RSYNC_ENABLED'] ?? 'true') !== 'false',
|
|
'weekly' => ($vars['WEEKLY_RSYNC_ENABLED'] ?? 'false') !== 'false',
|
|
'monthly' => ($vars['MONTHLY_RSYNC_ENABLED'] ?? 'true') !== 'false',
|
|
];
|
|
|
|
// Active rsync profiles — from lock files
|
|
$lockDir = '/tmp/unraid_locks';
|
|
$active = [];
|
|
foreach (glob("$lockDir/rsync_*.lock") ?: [] as $lf) {
|
|
$content = trim(@file_get_contents($lf) ?: '');
|
|
[$pid, $locked_name] = array_pad(explode(':', $content, 2), 2, '');
|
|
if (!$pid || !file_exists("/proc/$pid")) continue;
|
|
$profile = preg_replace('/^rsync_/', '', $locked_name ?: basename($lf, '.lock'));
|
|
$active[] = [
|
|
'profile' => $profile,
|
|
'pid' => (int)$pid,
|
|
'elapsed' => time() - (int)filemtime($lf),
|
|
];
|
|
}
|
|
|
|
// Last completed run per window (orchestrator log files)
|
|
$scriptMap = [
|
|
'critical' => 'critical_sync_maintenance',
|
|
'daily' => 'daily_sync_maintenance',
|
|
'intermediate' => 'intermediate_sync_maintenance',
|
|
'weekly' => 'weekly_sync_maintenance',
|
|
'monthly' => 'monthly_maintenance',
|
|
];
|
|
$lastSync = [];
|
|
foreach ($scriptMap as $key => $scriptName) {
|
|
$logFile = LOG_DIR . "/Orchestrators/$scriptName.json";
|
|
if (!file_exists($logFile)) continue;
|
|
$stat = json_decode(@file_get_contents($logFile) ?: '{}', true) ?: [];
|
|
$lastSync[$key] = [
|
|
'ts' => (int)($stat['end'] ?? $stat['start'] ?? 0),
|
|
'status' => $stat['status'] ?? 'unknown',
|
|
'duration' => isset($stat['start'], $stat['end'])
|
|
? (int)$stat['end'] - (int)$stat['start'] : null,
|
|
];
|
|
}
|
|
|
|
// Profile activity — last 7 days, aggregated per profile
|
|
$bwLog = DB_DIR . '/bandwidth_history.db';
|
|
$cutoff7 = date('Y-m-d', strtotime('-7 days'));
|
|
$profiles = [];
|
|
if (file_exists($bwLog)) {
|
|
foreach (file($bwLog, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ?: [] as $line) {
|
|
$p = explode('|', $line);
|
|
if (count($p) < 4 || ($p[0] ?? '') < $cutoff7) continue;
|
|
$name = $p[2] ?? '';
|
|
if (!$name || str_ends_with($name, '-fallback')) continue;
|
|
if (!isset($profiles[$name])) $profiles[$name] = ['runs' => 0, 'dur' => 0, 'bytes' => 0];
|
|
$profiles[$name]['runs']++;
|
|
$profiles[$name]['dur'] += (int)($p[3] ?? 0);
|
|
$profiles[$name]['bytes'] += (int)($p[5] ?? 0);
|
|
}
|
|
}
|
|
arsort($profiles); // sort by run count descending
|
|
$bwSummary = array_slice($profiles, 0, 6, true);
|
|
|
|
return [
|
|
'enabled' => $enabled,
|
|
'windows' => $windows,
|
|
'active' => $active,
|
|
'last_sync' => $lastSync,
|
|
'bw_summary' => $bwSummary,
|
|
];
|
|
}
|