Gmer4Lfe
331b7b6b13
MagicDNS does not work on this mesh, so `tailscale ip -4` missed and blocked on a system DNS lookup until it timed out: 5.04s per call against 0.010s for the status parse, paid by every mesh operation on both hosts. Exact match first, so no precision is traded for the speed.
1095 lines
59 KiB
PHP
1095 lines
59 KiB
PHP
<?php
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
// PURPOSE
|
|
// Root of the PHP layer. Locates the Varaverk installation, parses master.conf plus this
|
|
// host's own host*.conf into a flat array, and provides host identity, remote resolution,
|
|
// and the tmpfs payload cache. Every other include/ file requires this one.
|
|
//
|
|
// DESIGN PRINCIPLES
|
|
// varaverk.cfg is the single source of truth for location.
|
|
// SCRIPTS_DIR is read from it; CONF_DIR, DATA_DIR, STATE_DIR and DEPLOY_DIR are all
|
|
// derived. Storage-mode migration rewrites that one value and every path follows.
|
|
//
|
|
// Host identity mirrors bash detect_hosts() exactly.
|
|
// Same HOST<n> / HOST<n>_NAME matching, same 15-char NetBIOS truncation fallback. The
|
|
// two implementations must agree — a page that disagrees with the scripts about which
|
|
// host it is on is worse than one that cannot tell.
|
|
//
|
|
// Conf parsing resolves ${VAR} in two passes.
|
|
// Bash expands at runtime; PHP reads the file literally. Pass 1 substitutes
|
|
// ${SCRIPTS_DIR} from the PHP-side constant, pass 2 resolves remaining ${VAR} against
|
|
// the already-parsed set. Without this, every derived path arrives as a literal string.
|
|
//
|
|
// Read-only with respect to behaviour.
|
|
// This file parses conf and reports; it does not decide policy. Callers own that.
|
|
//
|
|
// OPERATIONAL SAFEGUARDS
|
|
// Ambiguous truncated hostnames are refused, never guessed.
|
|
// The 15-char fallback accepts a match only when exactly one configured host qualifies.
|
|
// Two plausible candidates return 'unknown' rather than picking one — a wrong host
|
|
// identity silently routes local work to a remote node.
|
|
//
|
|
// Cache writes are atomic.
|
|
// vv_cache_write() writes .tmp then rename()s into place, so a concurrent reader sees
|
|
// either the old payload or the new one, never a half-written file.
|
|
//
|
|
// Cache reads are age-gated and fail to null.
|
|
// Past $maxAge, vv_cache_read() returns null rather than stale data. Callers treat null
|
|
// as "no cache" and fall back to a live call — a missing cache can never be the reason
|
|
// a page fails to render.
|
|
//
|
|
// Unknown host degrades instead of guessing.
|
|
// vv_detect_host() returns 'unknown' and vv_conf_vars() then loads master.conf alone.
|
|
// Shared config still resolves; host-specific values are simply absent.
|
|
//
|
|
// EXPORTS
|
|
// Identity vv_detect_host(), vv_get_hostname(), vv_is_owner(), vv_is_ai_host(),
|
|
// vv_known_hosts()
|
|
// Config vv_conf_vars(), vv_read_conf_raw(), vv_write_conf_raw(), vv_get_conf_files()
|
|
// Parsing vv_parse_conf_scalar(), vv_parse_kv_db(), vv_format_uptime()
|
|
// Remote vv_resolve_tailscale_ip(), vv_remote_state_cmd(), vv_local_ip()
|
|
// Setup state vv_setup_state_read/_write(), vv_push_setup_state(), vv_push_master_conf()
|
|
// Cache vv_cache_read(), vv_cache_write()
|
|
// Unraid API vv_unraid_api_query(), vv_auto_create_api_key()
|
|
//
|
|
// CONFIGURATION
|
|
// varaverk.cfg SCRIPTS_DIR, CUSTOM_SCRIPTS_DIR
|
|
// master.conf HOST<n> / HOST<n>_NAME — host identity
|
|
// host*.conf HOST*_SSH_KEY — used for setup-state and conf push
|
|
// ident.cfg timeZone — Unraid's own setting, adopted for the whole PHP layer
|
|
// master.conf VV_CACHE_ROOT and the cache paths derived from it — shared with
|
|
// load_config.sh, which reads the same keys from the same file
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
// ── Timezone ──────────────────────────────────────────────────────────────────
|
|
// Set here, before anything else, because every PHP entry point in the plugin reaches this file
|
|
// and nothing that runs earlier has a date in it.
|
|
//
|
|
// PHP on Unraid has no date.timezone set and therefore runs in UTC, while the server itself runs
|
|
// local — America/New_York on this one. Every date this layer produced was consequently offset
|
|
// from every date the shell layer wrote, and the two are compared constantly: the token ledger,
|
|
// the rsync history, the arr run dates, the parity cron. The AI tab's "Today" tile read zero for
|
|
// the four hours between 8pm and midnight, the 7-day windows were cut at the wrong instant, and
|
|
// the next parity check was displayed four hours out. All of it silent, all of it in the same
|
|
// direction, all of it invisible for two thirds of the day.
|
|
//
|
|
// /etc/php.ini is the wrong place to fix it twice over: /etc is a RAM filesystem here, so the
|
|
// edit dies at the next reboot, and it would change the timezone for every other PHP application
|
|
// on the box to fix one of ours.
|
|
//
|
|
// ident.cfg is Unraid's own setting, on the flash, and is what the WebGUI's own clock uses — so
|
|
// this follows the operator's configured timezone rather than asserting one, and a second host
|
|
// in another zone gets its own. The /etc/localtime symlink is the fallback because it is what
|
|
// the C library actually honours; UTC last, which is merely today's broken behaviour made
|
|
// explicit rather than accidental.
|
|
$_vv_tz = '';
|
|
foreach (@file('/boot/config/ident.cfg') ?: [] as $_l) {
|
|
if (preg_match('/^\s*timeZone\s*=\s*"?([^"\r\n]+)"?/', $_l, $_m)) { $_vv_tz = trim($_m[1]); break; }
|
|
}
|
|
if ($_vv_tz === '') {
|
|
$_link = @readlink('/etc/localtime') ?: '';
|
|
if (preg_match('#zoneinfo/(.+)$#', $_link, $_m)) $_vv_tz = $_m[1];
|
|
}
|
|
// Validated before use. An unparseable value would otherwise raise a warning on every request
|
|
// and leave the process in UTC anyway, which is the bug this exists to remove.
|
|
if ($_vv_tz === '' || !@timezone_open($_vv_tz)) $_vv_tz = 'UTC';
|
|
date_default_timezone_set($_vv_tz);
|
|
define('VV_TIMEZONE', $_vv_tz);
|
|
unset($_vv_tz, $_l, $_m, $_link);
|
|
|
|
define('PLUGIN_CFG', '/boot/config/plugins/varaverk/varaverk.cfg');
|
|
|
|
$_vv_cfg = @parse_ini_file(PLUGIN_CFG) ?: [];
|
|
define('SCRIPTS_DIR', $_vv_cfg['SCRIPTS_DIR'] ?? '/boot/config/plugins/varaverk');
|
|
define('CONF_DIR', SCRIPTS_DIR . '/Configurations');
|
|
define('DEPLOY_DIR', SCRIPTS_DIR . '/Deployment');
|
|
// DATA_DIR is the one on-disk root; everything Varaverk persists lives under it in a
|
|
// subdirectory named for what the files are. Mirrors the same block in master.conf, which is
|
|
// where the shell layer reads them from — these are derived from SCRIPTS_DIR rather than parsed
|
|
// so that a conf that has not upgraded yet still resolves, and so this file keeps working when
|
|
// master.conf is missing entirely (setup, first boot, a botched pull).
|
|
//
|
|
// STATE_DIR moved from SCRIPTS_DIR/State_Files to DATA_DIR/state and kept its name, which is why
|
|
// the 23 call sites in this layer that build on it needed no edits at all.
|
|
// ── The clock this layer runs on ──────────────────────────────────────────────────────────────
|
|
// PHP on Unraid defaults to UTC while the host runs local time, and the two halves of Varaverk
|
|
// write into the same files. Every bash script stamps its log in local time; every PHP writer —
|
|
// the conf audit trail, the AI worker, the repair sweep, api/system.php — stamped UTC. On this
|
|
// host that is a four-hour disagreement inside conf_changes.log against every log you would
|
|
// correlate it with.
|
|
//
|
|
// The formatting was the visible half. The damaging half was parsing: vv_ai_syslog_ts() reads
|
|
// "Aug 14 19:00:01" out of /var/log/syslog, which the system wrote in local time, and handed it
|
|
// to strtotime() under UTC — landing every syslog event four hours earlier than it happened. The
|
|
// repair sweep bounds its scan to "since the last pass", so a fault that had just occurred could
|
|
// read as four hours old and fall outside the window entirely. It would have found nothing and
|
|
// said so honestly.
|
|
//
|
|
// Same correction fixes the date-string comparisons: bandwidth_history.db and arr_cleanup_stats.db
|
|
// are written by bash with local dates and compared against date('Y-m-d') here, which near
|
|
// midnight was a day out.
|
|
//
|
|
// Set from the host rather than hardcoded, and only when PHP has not been told otherwise, so an
|
|
// operator who deliberately configures a timezone keeps it.
|
|
if (!ini_get('date.timezone') || date_default_timezone_get() === 'UTC') {
|
|
$_vv_tz = @readlink('/etc/localtime') ?: '';
|
|
$_vv_p = strpos($_vv_tz, 'zoneinfo/');
|
|
if ($_vv_p !== false) {
|
|
$_vv_name = substr($_vv_tz, $_vv_p + 9);
|
|
// Validated against the real list — a malformed link must not leave the clock undefined.
|
|
if ($_vv_name !== '' && in_array($_vv_name, timezone_identifiers_list(), true)) {
|
|
date_default_timezone_set($_vv_name);
|
|
}
|
|
}
|
|
unset($_vv_tz, $_vv_p, $_vv_name);
|
|
}
|
|
|
|
define('DATA_DIR', SCRIPTS_DIR . '/data');
|
|
define('DB_DIR', DATA_DIR . '/db');
|
|
define('STATE_DIR', DATA_DIR . '/state');
|
|
define('AI_DATA_DIR', DATA_DIR . '/ai');
|
|
define('CACHE_BACKUP_DIR', DATA_DIR . '/cache');
|
|
define('LOG_ARCHIVE_DIR', DATA_DIR . '/logs');
|
|
define('BACKUP_DIR', DATA_DIR . '/Backups');
|
|
define('CONF_BACKUP_DIR', BACKUP_DIR . '/Confs');
|
|
define('LOG_DIR', '/var/log/varaverk');
|
|
// User-authored custom scripts (scheduler page "+ Create Script") — kept outside the git
|
|
// repo entirely, alongside the User Scripts plugin's own storage. Any *.sh file placed
|
|
// directly in this folder is auto-detected and listed — it doesn't have to be created
|
|
// through the page's editor.
|
|
define('CUSTOM_SCRIPTS_DIR', $_vv_cfg['CUSTOM_SCRIPTS_DIR'] ?? '/boot/config/plugins/user.scripts/Varaverk/Scripts');
|
|
unset($_vv_cfg);
|
|
|
|
define('VV_SETUP_STATE_FILE', STATE_DIR . '/varaverk_setup.db');
|
|
|
|
// ── The two install layouts ───────────────────────────────────────────────────
|
|
// The same pair storage_migrate.sh moves an installation between, and the same pair
|
|
// HOST*_STORAGE_MODE_INTERNAL selects: true → internal, false → flash. They are stated here
|
|
// because vv_push_master_conf() has to reason about a *remote* host's layout, where SCRIPTS_DIR
|
|
// describes this host and says nothing about the partner's — and about a partner that has no
|
|
// layout yet because Varaverk is not installed on it.
|
|
//
|
|
// Naming follows the conf, which names the boot device rather than the destination: "internal"
|
|
// is a host that boots from internal NVMe/SSD and can therefore keep everything on /boot;
|
|
// "flash" is a host booting from a USB stick, which must not take the writes, so its data lives
|
|
// in appdata instead.
|
|
define('VV_DIR_INTERNAL', '/boot/config/plugins/varaverk');
|
|
define('VV_DIR_FLASH', '/mnt/user/appdata/Varaverk');
|
|
|
|
// ── Cache roots ───────────────────────────────────────────────────────────────
|
|
// Read from master.conf so this layer and load_config.sh resolve the same paths from the same
|
|
// line. They used to be literals in three PHP files, restating what load_config.sh already said,
|
|
// because PHP cannot source bash — and the two sets were kept in agreement by hand.
|
|
//
|
|
// Deliberately NOT via vv_conf_vars(). That parses master.conf and this host's conf in full and
|
|
// calls vv_detect_host() on the way, and api/monitor.php's documented fast path loads this file
|
|
// and nothing else in order to reach vv_cache_read() on a cache hit. Making every request pay a
|
|
// full conf parse to learn a directory name would tax the exact path built to be cheap. One
|
|
// file, seven keys, no host detection.
|
|
//
|
|
// The fallbacks are the pre-consolidation paths, matching load_config.sh: an installation whose
|
|
// conf has not been through conf_upgrade yet keeps using what it is already using, rather than
|
|
// silently relocating its caches because a variable was missing.
|
|
$_vv_cache = [];
|
|
foreach (@file(CONF_DIR . '/master.conf') ?: [] as $_l) {
|
|
if (preg_match('/^\s*(VV_CACHE_ROOT|VV_CACHE_DIR|CONF_RAM_CACHE_DIR|ARR_CACHE_DIR'
|
|
. '|AI_TOKEN_CACHE_DIR|AI_JOB_DIR|DOCKER_JOB_DIR)\s*=\s*"?([^"#\r\n]+?)"?\s*(?:#.*)?$/',
|
|
$_l, $_m)) {
|
|
$_vv_cache[$_m[1]] = trim($_m[2]);
|
|
}
|
|
}
|
|
$_vv_root = $_vv_cache['VV_CACHE_ROOT'] ?? '/tmp/varaverk';
|
|
// Resolves the one reference the conf actually uses. This is not a general bash expander and is
|
|
// not trying to be — vv_conf_vars() owns that, and every value here is one level deep by design.
|
|
$_vv_path = function (string $key, string $fallback) use ($_vv_cache, $_vv_root): string {
|
|
$v = trim($_vv_cache[$key] ?? '');
|
|
if ($v === '') return $fallback;
|
|
return str_replace(['${VV_CACHE_ROOT}', '$VV_CACHE_ROOT'], $_vv_root, $v);
|
|
};
|
|
define('VV_CACHE_ROOT', $_vv_root);
|
|
define('VV_CACHE_DIR', $_vv_path('VV_CACHE_DIR', '/tmp/vv_cache'));
|
|
define('VV_CONF_RAM_CACHE_DIR', $_vv_path('CONF_RAM_CACHE_DIR', '/tmp/.cache/vv/d'));
|
|
define('VV_ARR_CACHE_DIR', $_vv_path('ARR_CACHE_DIR', '/tmp/arr_cache'));
|
|
define('VV_AI_TOKEN_CACHE_DIR', $_vv_path('AI_TOKEN_CACHE_DIR', '/tmp/.cache/vv/ai'));
|
|
define('VV_AI_JOB_DIR', $_vv_path('AI_JOB_DIR', '/tmp/varaverk_ai_jobs'));
|
|
define('VV_JOB_DIR', $_vv_path('DOCKER_JOB_DIR', '/tmp/varaverk_dk_jobs'));
|
|
unset($_vv_cache, $_vv_root, $_vv_path, $_l, $_m);
|
|
|
|
// Read the setup state file into a key=>value array.
|
|
function vv_setup_state_read(): array {
|
|
$out = [];
|
|
foreach (file(VV_SETUP_STATE_FILE) ?: [] as $line) {
|
|
[$k, $v] = explode('=', trim($line), 2) + ['', ''];
|
|
if ($k !== '') $out[$k] = $v;
|
|
}
|
|
return $out;
|
|
}
|
|
|
|
// Write the setup state file (creates or overwrites).
|
|
// Returns whether the write landed. Was void, so a caller had no way to tell a full flash or a
|
|
// read-only mount from success — and `if (!vv_setup_state_write(...))` on a void function is
|
|
// always true, which turns "it worked" into a reported failure. Existing void callers are
|
|
// unaffected by the added return.
|
|
function vv_setup_state_write(array $data): bool {
|
|
$content = '';
|
|
foreach ($data as $k => $v) $content .= "$k=$v\n";
|
|
return file_put_contents(VV_SETUP_STATE_FILE, $content) !== false;
|
|
}
|
|
|
|
// Push the setup state file to all remote hosts via scp.
|
|
// Reads the remote's varaverk.cfg to find their actual SCRIPTS_DIR (handles appdata mode).
|
|
//
|
|
// Returns the number of hosts the push FAILED to reach — 0 when every partner took it, and 0
|
|
// when there was nothing to do. This was `: void`, and it threw away the exec() status too, so
|
|
// the adapter that wraps it was documented "Always returns 0" and every caller testing it was
|
|
// testing a constant. partnership_offboard.sh reported "Phase flags pushed ✅" on runs whose own
|
|
// body had just warned the push failed, which is the difference between a mirror that knows the
|
|
// partnership ended and one whose wizard still shows a finished onboard.
|
|
//
|
|
// A host with no resolvable Tailscale IP counts as a failure, not a skip: it is precisely the
|
|
// case where the partner keeps stale state and nobody is told.
|
|
function vv_push_setup_state(): int {
|
|
if (!file_exists(VV_SETUP_STATE_FILE)) return 0;
|
|
$myHostId = vv_detect_host();
|
|
$vars = vv_conf_vars();
|
|
$sshKey = $vars[strtoupper($myHostId) . '_SSH_KEY'] ?? '';
|
|
if (!$sshKey || !file_exists($sshKey)) return 0;
|
|
$failed = 0;
|
|
|
|
$master = vv_read_conf_raw('master.conf');
|
|
preg_match_all('/^\s*(HOST\d+)(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m);
|
|
$seen = [];
|
|
foreach ($m[1] as $i => $hostKey) {
|
|
$hostId = strtolower($hostKey);
|
|
if ($hostId === $myHostId || isset($seen[$hostId])) continue;
|
|
$seen[$hostId] = true;
|
|
$hostname = trim($m[2][$i]);
|
|
if (!$hostname) continue;
|
|
$ip = vv_resolve_tailscale_ip($hostname);
|
|
if (!$ip) { $failed++; continue; }
|
|
$sshBase = 'ssh -i ' . escapeshellarg($sshKey)
|
|
. ' -o ConnectTimeout=10 -o StrictHostKeyChecking=no -o BatchMode=yes root@' . $ip;
|
|
|
|
// Get remote SCRIPTS_DIR from varaverk.cfg — handles appdata mode on remote.
|
|
// Falls back to the default install path if varaverk.cfg is absent (pre-install).
|
|
$cfgRaw = trim(shell_exec($sshBase . ' "cat /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null"') ?: '');
|
|
$remoteSD = '/boot/config/plugins/varaverk';
|
|
foreach (explode("\n", $cfgRaw) as $line) {
|
|
if (str_starts_with(trim($line), 'SCRIPTS_DIR=')) {
|
|
$remoteSD = trim(substr(trim($line), strlen('SCRIPTS_DIR=')), '"\'');
|
|
break;
|
|
}
|
|
}
|
|
// Which layout the partner uses is decided ON the partner, not assumed here. State moved
|
|
// from SCRIPTS_DIR/State_Files to DATA_DIR/state, and this file is written to whichever
|
|
// one that host will actually read — a setup state pushed to the directory the partner
|
|
// does not read is worse than not pushing it, because the push reports success.
|
|
//
|
|
// The order matters: prefer the new path, fall back to the old ONLY if it already exists.
|
|
// A partner that has neither is a fresh install on current code, which reads the new one.
|
|
// Piped over ssh rather than scp'd so the resolution and the write are the same call —
|
|
// scp needs the path decided here, which is the thing that cannot be known here.
|
|
$remoteResolve = 'sf="' . $remoteSD . '/data/state"; '
|
|
. '[ -d "$sf" ] || { [ -d "' . $remoteSD . '/State_Files" ] '
|
|
. '&& sf="' . $remoteSD . '/State_Files"; }; '
|
|
. 'mkdir -p "$sf" && cat > "$sf/varaverk_setup.db"';
|
|
// 2>&1 without capturing the status was the whole problem: the command could fail on an
|
|
// unreachable host, a full disk or a refused key and this function returned identically.
|
|
$out = [];
|
|
$rc = 0;
|
|
exec('cat ' . escapeshellarg(VV_SETUP_STATE_FILE) . ' | '
|
|
. $sshBase . ' ' . escapeshellarg($remoteResolve) . ' 2>&1', $out, $rc);
|
|
if ($rc !== 0) $failed++;
|
|
}
|
|
return $failed;
|
|
}
|
|
|
|
// Push master.conf to all remote hosts via scp after a local save.
|
|
// Returns one result entry per remote found in master.conf.
|
|
// Silently returns [] on non-owner hosts (no SSH key, no remote access).
|
|
function vv_push_master_conf(): array {
|
|
$myHostId = vv_detect_host();
|
|
$vars = vv_conf_vars();
|
|
$sshKey = $vars[strtoupper($myHostId) . '_SSH_KEY'] ?? '';
|
|
if (!$sshKey || !file_exists($sshKey)) return [];
|
|
|
|
$localPath = CONF_DIR . '/master.conf';
|
|
$master = vv_read_conf_raw('master.conf');
|
|
preg_match_all('/^\s*(HOST\d+)(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m);
|
|
|
|
$results = [];
|
|
$seen = [];
|
|
foreach ($m[1] as $i => $hostKey) {
|
|
$hostId = strtolower($hostKey);
|
|
if ($hostId === $myHostId || isset($seen[$hostId])) continue;
|
|
$seen[$hostId] = true;
|
|
|
|
$hostname = trim($m[2][$i]);
|
|
$ip = vv_resolve_tailscale_ip($hostname);
|
|
if (!$ip) {
|
|
$results[] = ['host' => $hostKey, 'ok' => false, 'ready' => false, 'error' => 'Tailscale IP not found'];
|
|
continue;
|
|
}
|
|
|
|
// Single SSH call decides which directory on the remote receives the conf. Two layouts
|
|
// exist and a third state — not installed yet — is the one this has to serve first.
|
|
//
|
|
// 1. varaverk.cfg names SCRIPTS_DIR and that install has a Configurations/ — the
|
|
// remote has told us where it lives, and it outranks anything found by looking.
|
|
// A node migrated to appdata can still have a stale master.conf on /boot; picking
|
|
// the file over the declaration would write to the copy nothing reads.
|
|
// 2. no varaverk.cfg — look for an existing master.conf in the internal layout, then
|
|
// the flash layout (the same two roots storage_migrate.sh moves between). A
|
|
// half-installed or part-migrated node is found this way.
|
|
// 3. neither — SEED. mkdir the internal path and deliver there.
|
|
//
|
|
// Case 3 is the point of the whole function during Phase 1 of onboarding. The partner
|
|
// has no Varaverk at all yet, and the conf is what tells its installer who HOST1 and
|
|
// HOST2 are, so the wizard can skip the identity questions it would otherwise ask about
|
|
// a mesh it is already a member of. Refusing to push until the plugin was installed
|
|
// made that impossible: the information had to arrive first to be useful.
|
|
//
|
|
// Internal is the seed target because /boot is mounted before the array is and the .plg
|
|
// installs there unconditionally — appdata may not exist yet on a node whose array has
|
|
// never started. If the operator then chooses flash in the wizard, storage_migrate.sh
|
|
// carries the conf across with the rest of the install.
|
|
//
|
|
// The .plg only seeds master.conf from the template when none is present, so a conf
|
|
// delivered ahead of the install survives it, as does `git reset --hard` — every conf
|
|
// is gitignored and therefore untracked.
|
|
//
|
|
// NOTE: this sends the owner's master.conf as-is, credentials included — TAILSCALE_API_KEY
|
|
// and WEBHOOK_SECRET among them. That is the accepted trade for one shared config across
|
|
// the mesh; a node you would not hand those to should not be a listed host.
|
|
// Remote command built as one PHP string and escapeshellarg()'d whole — shell_exec()
|
|
// adds its own `sh -c` layer locally, so a bare double-quoted string here would let
|
|
// the $(...)/${...} substitutions expand on HOST1 before ssh ever sees them, instead
|
|
// of on the remote host. escapeshellarg() keeps it opaque until the remote shell runs it.
|
|
$sshBase = 'ssh -i ' . escapeshellarg($sshKey)
|
|
. ' -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@' . $ip;
|
|
$remoteCmd = 'sd=$(grep -oP \'(?<=SCRIPTS_DIR=")[^"]+\' /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null); '
|
|
. 'd=""; '
|
|
. '[ -n "$sd" ] && [ -d "$sd/Configurations" ] && d="$sd/Configurations"; '
|
|
. '[ -z "$d" ] && [ -f "' . VV_DIR_INTERNAL . '/Configurations/master.conf" ] && d="' . VV_DIR_INTERNAL . '/Configurations"; '
|
|
. '[ -z "$d" ] && [ -f "' . VV_DIR_FLASH . '/Configurations/master.conf" ] && d="' . VV_DIR_FLASH . '/Configurations"; '
|
|
. '[ -z "$d" ] && { mkdir -p "' . VV_DIR_INTERNAL . '/Configurations" || exit 1; d="' . VV_DIR_INTERNAL . '/Configurations"; }; '
|
|
. 'if [ -f "$d/master.conf" ]; then echo "$d|update"; else echo "$d|seed"; fi';
|
|
// Last non-empty line only. Anything the remote's login shell prints of its own accord —
|
|
// a profile banner, an MOTD echoed to stdout — arrives ahead of the answer, and taking
|
|
// the whole output would build an scp destination out of it.
|
|
$probeRaw = trim(shell_exec($sshBase . ' ' . escapeshellarg($remoteCmd)) ?: '');
|
|
$probeLines = array_filter(array_map('trim', explode("\n", $probeRaw)), fn($l) => $l !== '');
|
|
$probe = $probeLines ? end($probeLines) : '';
|
|
|
|
if ($probe === '' || !str_contains($probe, '|')) {
|
|
$results[] = ['host' => $hostKey, 'ok' => false, 'ready' => false,
|
|
'error' => 'no usable conf directory on the remote — SSH failed or /boot is not writable'];
|
|
continue;
|
|
}
|
|
|
|
[$remoteConf, $mode] = explode('|', $probe, 2);
|
|
$dest = escapeshellarg('root@' . $ip . ':' . $remoteConf . '/master.conf');
|
|
$cmd = 'scp -i ' . escapeshellarg($sshKey)
|
|
. ' -o ConnectTimeout=10 -o StrictHostKeyChecking=no'
|
|
. ' ' . escapeshellarg($localPath) . ' ' . $dest . ' 2>&1';
|
|
// exec() APPENDS to its output array. Left uncleared, a failure on one host would be
|
|
// reported again as part of the next host's error.
|
|
$out = [];
|
|
exec($cmd, $out, $rc);
|
|
$results[] = [
|
|
'host' => $hostKey,
|
|
'ok' => $rc === 0,
|
|
'mode' => $mode, // 'seed' = delivered where there was nothing, 'update' = replaced
|
|
'path' => $remoteConf,
|
|
'error' => $rc !== 0 ? implode('; ', $out) : '',
|
|
];
|
|
}
|
|
return $results;
|
|
}
|
|
|
|
// Cached: this forks a shell, and it is reached from vv_conf_vars() by way of vv_detect_host(),
|
|
// which the repair sweep calls per log line. A machine does not rename itself mid-request.
|
|
function vv_get_hostname(bool $flush = false): string {
|
|
static $name = null;
|
|
if ($flush) { $name = null; return ''; }
|
|
if ($name === null) $name = trim(shell_exec('hostname -s') ?: '');
|
|
return $name;
|
|
}
|
|
|
|
// Mirror of common.sh resolve_tailscale_ip(): resolves a hostname to a Tailscale IPv4.
|
|
//
|
|
// `tailscale status` is asked first, and that ordering is the whole point. It used to try
|
|
// `tailscale ip -4` first, on the reasoning that Tailscale owns the mapping — true, but that call
|
|
// resolves through MagicDNS, and MagicDNS does not work here: these hosts sit on separate tailnets
|
|
// shared into each other, so the name misses and the call falls through to a system DNS lookup
|
|
// that times out. Measured at **5.04 seconds, every call**, against 0.010s for the status parse.
|
|
//
|
|
// Nine files resolve peers this way — the AI RPC, node_chat, the arr collector, the media mesh
|
|
// view — so that was five seconds added to every mesh operation on both hosts, quietly, for as
|
|
// long as the mesh has existed. Nothing looked broken; everything was just slow.
|
|
//
|
|
// Order is now: exact match, unambiguous prefix match, then the DNS path bounded to two seconds as
|
|
// a last resort. Exactness is not given up to get the speed — `tailscale status` carries the same
|
|
// mapping `tailscale ip` would return, and an exact name match against it is exactly as precise.
|
|
//
|
|
// Memoised per request: a page that resolves the same partner four times paid four lookups.
|
|
function vv_resolve_tailscale_ip(string $hostname): string {
|
|
static $cache = [];
|
|
$h = strtolower($hostname);
|
|
if (isset($cache[$h])) return $cache[$h];
|
|
|
|
$out = shell_exec('tailscale status 2>/dev/null') ?: '';
|
|
$matches = [];
|
|
foreach (explode("\n", $out) as $line) {
|
|
$cols = preg_split('/\s+/', trim($line));
|
|
if (!isset($cols[1])) continue;
|
|
$peerHost = strtolower(explode('.', $cols[1])[0]);
|
|
|
|
if ($peerHost === $h) return $cache[$h] = $cols[0];
|
|
|
|
// Unambiguous prefix match, either direction — handles Unraid's 15-char NetBIOS hostname
|
|
// truncation against a longer name recorded in master.conf. Only accepted when exactly one
|
|
// peer could qualify; never a guess between candidates sharing a prefix (server1/server10).
|
|
if (str_starts_with($peerHost, $h) || str_starts_with($h, $peerHost)) $matches[] = $cols[0];
|
|
}
|
|
if (count($matches) === 1) return $cache[$h] = $matches[0];
|
|
|
|
// Only reached when status cannot decide. Bounded, because this is the path that blocks on DNS
|
|
// when MagicDNS is unavailable — which is the normal case on this mesh.
|
|
return $cache[$h] = trim(shell_exec('timeout 2 tailscale ip -4 ' . escapeshellarg($h) . ' 2>/dev/null') ?: '');
|
|
}
|
|
|
|
// Cached alongside the others: this reads master.conf in full and is called by vv_conf_vars() on
|
|
// every lookup, so leaving it uncached would mean the conf is still read once per key even with
|
|
// the parsed values cached.
|
|
function vv_detect_host(bool $flush = false): string {
|
|
static $host = null;
|
|
if ($flush) { $host = null; return ''; }
|
|
if ($host !== null) return $host;
|
|
|
|
$host = _vv_detect_host_uncached();
|
|
return $host;
|
|
}
|
|
|
|
function _vv_detect_host_uncached(): string {
|
|
// Reads master.conf for HOST1="name" (or HOST1_NAME="name") and matches running hostname.
|
|
// Returns 'host1', 'host2', 'host3', ... or 'unknown'. Works for any number of hosts.
|
|
$master = vv_read_conf_raw('master.conf');
|
|
preg_match_all('/^\s*(HOST\d+)(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m);
|
|
$hostname = vv_get_hostname();
|
|
foreach ($m[1] as $i => $key) {
|
|
if (strcasecmp($hostname, trim($m[2][$i])) === 0) return strtolower($key);
|
|
}
|
|
// Unraid truncates the Server Name to 15 chars (NetBIOS limit). If the live hostname
|
|
// is at that exact limit, the name configured in master.conf (matching what Tailscale
|
|
// shows for this peer — resolve_tailscale_ip() keys off the same value) may be a longer,
|
|
// untruncated version. Only accept the match when it's unambiguous — exactly one
|
|
// configured host may qualify; never guess between multiple prefix candidates.
|
|
if (strlen($hostname) === 15) {
|
|
$candidates = [];
|
|
foreach ($m[1] as $i => $key) {
|
|
$configured = trim($m[2][$i]);
|
|
if (strlen($configured) > 15 && stripos($configured, $hostname) === 0) $candidates[] = $key;
|
|
}
|
|
if (count($candidates) === 1) return strtolower($candidates[0]);
|
|
}
|
|
return 'unknown';
|
|
}
|
|
|
|
function vv_is_owner(): bool {
|
|
return vv_detect_host() === 'host1';
|
|
}
|
|
|
|
// The AI subsystem is HOST1-only: it is the node with the GPU, the Ollama process and the
|
|
// index. Deliberately a separate predicate from vv_is_owner() even though both resolve to
|
|
// host1 today — one says "auth source of truth", this one says "AI runs here", and the day
|
|
// either moves, conflating them would move the other by accident.
|
|
//
|
|
// Returns false for 'unknown', so a host that cannot identify itself never shows the tab.
|
|
// Which node runs the model, the index and the bug store. Declared, not assumed: it was
|
|
// `=== 'host1'` for as long as host1 was the only box with a GPU, which made a physical fact look
|
|
// like a rule. A mesh can put the card anywhere — a spare GPU on someone else's node, a rebuilt
|
|
// host3 — and the owner has to be able to move without editing PHP.
|
|
//
|
|
// Defaults to host1 because whoever builds the mesh is host1, and is the person who would be
|
|
// changing this if it were ever wrong.
|
|
function vv_ai_owner_host(): string {
|
|
$h = strtolower(trim((string)(vv_conf_vars()['AI_OWNER_HOST'] ?? '')));
|
|
return preg_match('/^host\d+$/', $h) ? $h : 'host1';
|
|
}
|
|
|
|
function vv_ai_is_owner(): bool {
|
|
return vv_detect_host() === vv_ai_owner_host();
|
|
}
|
|
|
|
// Whose Ollama this node should talk to: its own if it has one, otherwise the owner's.
|
|
//
|
|
// "Scans local, then defaults to the mesh" — a node with a GPU uses it, and a node without one
|
|
// borrows rather than going without. The check is whether the conf declares a URL for this host,
|
|
// not whether Ollama answers: a reachability probe on every call would spend a network round trip
|
|
// to decide where to send a network round trip, and a box whose own model is down wants that said
|
|
// plainly rather than papered over by silently using someone else's.
|
|
// Every node that declares a model, in the order this one should try them: itself first, then the
|
|
// owner, then anyone else. Local before mesh because a GPU you already have costs nothing to
|
|
// reach; owner before the rest because that is where the index and the curated model live.
|
|
function vv_ai_model_candidates(): array {
|
|
$vars = vv_conf_vars();
|
|
$me = vv_detect_host();
|
|
$owner = vv_ai_owner_host();
|
|
$order = array_unique(array_merge([$me, $owner], array_keys(vv_known_hosts())));
|
|
|
|
$out = [];
|
|
foreach ($order as $h) {
|
|
if (trim((string)($vars[strtoupper($h) . '_OLLAMA_URL'] ?? '')) !== '') $out[] = $h;
|
|
}
|
|
return $out;
|
|
}
|
|
|
|
// Where the resolved choice is remembered. VV_CACHE_ROOT is tmpfs, which is the right lifetime:
|
|
// a reboot re-resolves, and nothing about which node answered belongs on flash.
|
|
function vv_ai_model_pin_path(): string {
|
|
return rtrim(VV_CACHE_DIR, '/') . '/ai_model_host.json';
|
|
}
|
|
|
|
// The node whose model this one uses. Resolved once, then pinned until something fails.
|
|
//
|
|
// Pinned rather than re-derived because resolution is only interesting when it changes: a mesh
|
|
// where every request re-decides which node to ask is a mesh that will eventually decide
|
|
// differently mid-conversation, and a chat whose second turn goes to another machine has no
|
|
// history there. The pin is what makes "borrow the owner's model" a stable answer.
|
|
//
|
|
// Cleared by vv_ai_model_failed(), never by a timer. A working endpoint does not need rechecking,
|
|
// and an expiry would reintroduce exactly the mid-conversation switch the pin exists to stop.
|
|
function vv_ai_model_host(): string {
|
|
$cands = vv_ai_model_candidates();
|
|
if (!$cands) return vv_ai_owner_host();
|
|
|
|
$pin = @json_decode((string)@file_get_contents(vv_ai_model_pin_path()), true);
|
|
if (is_array($pin) && in_array($pin['host'] ?? '', $cands, true)) return $pin['host'];
|
|
|
|
$chosen = $cands[0];
|
|
@file_put_contents(vv_ai_model_pin_path(),
|
|
json_encode(['host' => $chosen, 'at' => time()]), LOCK_EX);
|
|
return $chosen;
|
|
}
|
|
|
|
// Called when a model call fails. Drops the pin and, if there is somewhere else to go, records the
|
|
// failed node so the next resolution steps past it rather than pinning it again.
|
|
//
|
|
// Deliberately forgets the failure as soon as the alternatives run out: a single-model mesh whose
|
|
// only node is briefly down should keep pointing at it and report that it is down, not resolve to
|
|
// nothing and report that AI is unconfigured. Those are different problems and only one of them is
|
|
// the operator's to fix.
|
|
function vv_ai_model_failed(string $host = ''): void {
|
|
$host = $host !== '' ? $host : vv_ai_model_host();
|
|
$cands = vv_ai_model_candidates();
|
|
$rest = array_values(array_filter($cands, fn($h) => $h !== $host));
|
|
|
|
if (!$rest) { @unlink(vv_ai_model_pin_path()); return; }
|
|
@file_put_contents(vv_ai_model_pin_path(),
|
|
json_encode(['host' => $rest[0], 'at' => time(), 'after_failure' => $host]), LOCK_EX);
|
|
}
|
|
|
|
// Retained as the owner test it always was, so nothing that meant "is this the AI node" changes
|
|
// meaning underneath. Callers that meant "may this node use AI at all" want vv_ai_ui_on().
|
|
function vv_is_ai_host(): bool {
|
|
return vv_ai_is_owner();
|
|
}
|
|
|
|
// Whether the UI may offer anything AI at all: the right host, with the master switch on. Every
|
|
// AI surface asks this one question — the AI tab, the assistant dock on the Scheduler, and the
|
|
// two AI rows on the Tools card — so AI off means AI gone, not gone from most places.
|
|
//
|
|
// It lives here rather than in include/ai.php because the pages that need it do not all load
|
|
// that file; the Scheduler loads only config.php, and a gate that silently answers false where
|
|
// its definition is missing is worse than no gate. Reads AI_ENABLED directly for the same
|
|
// reason. Fail-closed on anything but the literal "true", matching the conf's own contract.
|
|
// May this node show AI features — the assistant docks, the findings strips, the AI rows on the
|
|
// Monitor card. No longer "am I the owner": the mesh shares one AI, and every node reaches it
|
|
// through include/ai_rpc.php. What a mirror does not get is the AI tab; see vv_ai_owner_ui_on().
|
|
//
|
|
// The two halves are asymmetric on purpose. The owner still fails closed on its own URL, because
|
|
// a missing local URL there is a conf error nothing can work around. A mirror does not test
|
|
// reachability at all: it is one SSH round trip away from the answer, on a link that goes down
|
|
// and comes back, and hiding the entire assistant on a transient blip is worse than showing it
|
|
// and reporting the failure in place. vv_ai_rpc() names every transport failure precisely so this
|
|
// gate does not have to guess at one.
|
|
//
|
|
// It lives here rather than in include/ai.php because the pages that need it do not all load that
|
|
// file; the Scheduler loads only config.php, and a gate that silently answers false where its
|
|
// definition is missing is worse than no gate.
|
|
function vv_ai_ui_on(): bool {
|
|
if (strtolower(trim(vv_conf_vars()['AI_ENABLED'] ?? 'false')) !== 'true') return false;
|
|
|
|
if (vv_ai_is_owner()) {
|
|
return trim((string)(vv_conf_vars()[strtoupper(vv_ai_owner_host()) . '_OLLAMA_URL'] ?? '')) !== '';
|
|
}
|
|
// A mirror needs somewhere to send the request. Without a hostname for the owner there is no
|
|
// round trip to attempt, and that is a conf gap rather than a transient one.
|
|
return trim((string)(vv_conf_vars()[strtoupper(vv_ai_owner_host())] ?? '')) !== '';
|
|
}
|
|
|
|
// May this node show the AI tab. Owner only, and deliberately so: that page carries the bug
|
|
// reports, the retrieval index and the model configuration — the surface where a wrong answer is
|
|
// expensive and the vocabulary assumes you built the thing. Someone running two containers on a
|
|
// node they were handed should have the assistant, not the machinery behind it.
|
|
function vv_ai_owner_ui_on(): bool {
|
|
return vv_ai_is_owner()
|
|
&& strtolower(trim(vv_conf_vars()['AI_ENABLED'] ?? 'false')) === 'true';
|
|
}
|
|
|
|
function vv_read_conf_raw(string $filename): string {
|
|
$path = CONF_DIR . '/' . $filename;
|
|
return file_exists($path) ? file_get_contents($path) : '';
|
|
}
|
|
|
|
function vv_write_conf_raw(string $filename, string $content): bool {
|
|
$path = CONF_DIR . '/' . $filename;
|
|
$tmp = $path . '.vv.tmp';
|
|
|
|
// Guarantee the trailing newline. A textarea does not supply one, so saving through the raw
|
|
// editor left master.conf ending mid-line — and appending is a real pattern here
|
|
// (conf_upgrade, and the array writers below), so the next `>>` would have joined itself
|
|
// onto the last setting instead of starting a line. Appended, never stripped: a deliberate
|
|
// run of blank lines at the end of a conf is the author's business.
|
|
if ($content !== '' && !str_ends_with($content, "\n")) $content .= "\n";
|
|
|
|
if (file_put_contents($tmp, $content) === false) return false;
|
|
if (!rename($tmp, $path)) return false;
|
|
|
|
// Every conf write in the plugin lands here, so this is the one place the parsed-conf cache
|
|
// has to be dropped. Doing it in the callers instead would mean a new writer inheriting a
|
|
// stale cache and no obvious reason why.
|
|
vv_conf_vars_flush();
|
|
return true;
|
|
}
|
|
|
|
function vv_get_conf_files(): array {
|
|
// Returns conf files this host is allowed to view/edit
|
|
$host = vv_detect_host();
|
|
$files = [];
|
|
if ($host === 'host1') {
|
|
// Owner sees master.conf + their own host conf
|
|
$files[] = 'master.conf';
|
|
$files[] = 'host1.conf';
|
|
} elseif (preg_match('/^host(\d+)$/', $host)) {
|
|
// Any other numbered host sees only their own conf
|
|
$files[] = $host . '.conf';
|
|
} else {
|
|
// Unknown host — show all for dev/debug
|
|
foreach (glob(CONF_DIR . '/*.conf') as $f) {
|
|
$files[] = basename($f);
|
|
}
|
|
}
|
|
return $files;
|
|
}
|
|
|
|
// Drops the parsed-conf cache. Called by vv_write_conf_raw() — the single point at which a conf
|
|
// changes on disk — so no writer has to remember to do it.
|
|
//
|
|
// Necessary on top of the mtime check below, not instead of it: filesystem mtimes have one-second
|
|
// resolution, and a write followed by a read inside the same second is exactly what the conf
|
|
// writer does when it verifies a value it just wrote. Without this, that read could be served the
|
|
// value from before the write and the verification would compare a value against itself.
|
|
function vv_conf_vars_flush(): void {
|
|
vv_conf_vars(true);
|
|
// Host identity is derived from master.conf too, so a write that changes HOST1 has to
|
|
// invalidate it as well — first-run setup does exactly that, then asks which host this is.
|
|
vv_detect_host(true);
|
|
vv_get_hostname(true);
|
|
}
|
|
|
|
// Parse conf into key=>value map for $VAR substitution in docs.
|
|
//
|
|
// Cached, because this is not the cheap function its callers assume. It reads and regex-parses
|
|
// both conf files on every call — around 1,900 lines — and the repair sweep asks it for a key per
|
|
// log line. The measured cost of exactly this pattern is on record: the 30 minutes play_state_sync
|
|
// spent in 2026-07 was parse and fork overhead, not the API it was blamed on.
|
|
//
|
|
// Keyed on a hash of the contents, so an edit made outside this process — the raw editor in
|
|
// another tab, conf_upgrade from bash, a hand edit over SSH — is still picked up.
|
|
//
|
|
// Content rather than mtime and size, which was the first attempt and was wrong. Both conf files
|
|
// were rewritten within the same second and to the same byte count — "true" and "false" trading
|
|
// places across two keys — and the stale values were served straight back. mtime has one-second
|
|
// resolution and a same-size edit is not a rare shape in a file of booleans.
|
|
//
|
|
// Reading both files every call is not what costs anything here; parsing them is. The read is
|
|
// tens of microseconds against a parse of ~1,900 lines followed by two resolution passes.
|
|
function vv_conf_vars(bool $flush = false): array {
|
|
static $cache = null;
|
|
static $stamp = null;
|
|
|
|
if ($flush) { $cache = null; $stamp = null; return []; }
|
|
|
|
$files = ['master.conf'];
|
|
$host = vv_detect_host();
|
|
if (preg_match('/^host\d+$/', $host)) $files[] = $host . '.conf';
|
|
|
|
$raws = [];
|
|
$sig = '';
|
|
foreach ($files as $f) {
|
|
$raws[$f] = vv_read_conf_raw($f);
|
|
$sig .= md5($raws[$f]);
|
|
}
|
|
if ($cache !== null && $stamp === $sig) return $cache;
|
|
|
|
$vars = [];
|
|
foreach ($files as $f) {
|
|
$raw = $raws[$f];
|
|
// Match: VAR_NAME="value" or VAR_NAME=value (no quotes)
|
|
preg_match_all('/^\s*([A-Z0-9_]+)\s*=\s*["\']?([^"\'#\n]*?)["\']?\s*(?:#.*)?$/m', $raw, $m);
|
|
foreach ($m[1] as $i => $key) {
|
|
$vars[$key] = str_replace('\\$', '$', trim($m[2][$i]));
|
|
}
|
|
}
|
|
// Resolve bash variable references — bash expands ${VAR} at runtime; PHP reads them literally.
|
|
// Pass 1: ${SCRIPTS_DIR} from the PHP-computed constant (other vars depend on it).
|
|
// Pass 2: ${VAR} using now-resolved values from within the same conf set.
|
|
foreach ($vars as $k => &$v) {
|
|
if (is_string($v)) $v = str_replace('${SCRIPTS_DIR}', SCRIPTS_DIR, $v);
|
|
}
|
|
foreach ($vars as $k => &$v) {
|
|
if (is_string($v) && str_contains($v, '${')) {
|
|
$v = preg_replace_callback('/\$\{([A-Z0-9_]+)\}/', function ($m) use ($vars) {
|
|
return $vars[$m[1]] ?? $m[0];
|
|
}, $v);
|
|
}
|
|
}
|
|
unset($v);
|
|
|
|
$stamp = $sig;
|
|
return $cache = $vars;
|
|
}
|
|
|
|
// Query the Unraid GraphQL API for a given host.
|
|
// For the local host queries http://localhost/graphql; for remote hosts uses the Tailscale IP.
|
|
// $apiKey may be passed explicitly (needed when querying a remote host from the local host,
|
|
// since vv_conf_vars() only loads the current host's conf file).
|
|
// Returns the decoded 'data' object on success, null on any failure.
|
|
// Debug log written to /tmp/vv_api_debug.json on failure.
|
|
function vv_unraid_api_query(string $hostId, string $gql, int $timeoutSec = 5, string $apiKey = ''): ?array {
|
|
$vars = vv_conf_vars();
|
|
$key = $apiKey ?: ($vars[strtoupper($hostId) . '_UNRAID_API_KEY'] ?? '');
|
|
if (!$key) return null;
|
|
|
|
$myHostId = vv_detect_host();
|
|
if (strtolower($hostId) === strtolower($myHostId)) {
|
|
$url = 'http://localhost/graphql';
|
|
} else {
|
|
$hostname = $vars[strtoupper($hostId)] ?? '';
|
|
if (!$hostname) return null;
|
|
$ip = vv_resolve_tailscale_ip($hostname);
|
|
if (!$ip) return null;
|
|
$url = "http://{$ip}/graphql";
|
|
}
|
|
|
|
$body = json_encode(['query' => $gql]);
|
|
|
|
// Use curl (preferred — doesn't require allow_url_fopen, better error handling).
|
|
if (function_exists('curl_init')) {
|
|
$ch = curl_init($url);
|
|
curl_setopt_array($ch, [
|
|
CURLOPT_POST => true,
|
|
CURLOPT_HTTPHEADER => ['Content-Type: application/json', "x-api-key: {$key}"],
|
|
CURLOPT_POSTFIELDS => $body,
|
|
CURLOPT_RETURNTRANSFER => true,
|
|
CURLOPT_TIMEOUT => $timeoutSec,
|
|
CURLOPT_CONNECTTIMEOUT => 3,
|
|
CURLOPT_FOLLOWLOCATION => false,
|
|
]);
|
|
$resp = curl_exec($ch);
|
|
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
|
$curlErr = curl_error($ch);
|
|
curl_close($ch);
|
|
} else {
|
|
// Fallback to file_get_contents if curl is unavailable.
|
|
$ctx = stream_context_create(['http' => [
|
|
'method' => 'POST',
|
|
'header' => "Content-Type: application/json\r\nx-api-key: {$key}",
|
|
'content' => $body,
|
|
'timeout' => $timeoutSec,
|
|
'ignore_errors' => true,
|
|
]]);
|
|
$resp = @file_get_contents($url, false, $ctx);
|
|
$httpCode = $resp !== false ? 200 : 0;
|
|
$curlErr = '';
|
|
}
|
|
|
|
if ($resp === false || $resp === '' || ($httpCode !== 0 && $httpCode !== 200)) {
|
|
@file_put_contents(VV_CACHE_DIR . '/vv_api_debug.json', json_encode([
|
|
'ts' => time(),
|
|
'host' => $hostId,
|
|
'url' => $url,
|
|
'http_code' => $httpCode,
|
|
'curl_err' => $curlErr,
|
|
'response' => substr((string)$resp, 0, 800),
|
|
], JSON_PRETTY_PRINT));
|
|
return null;
|
|
}
|
|
|
|
$decoded = json_decode((string)$resp, true);
|
|
|
|
// If the API returned GraphQL errors, log them for diagnosis.
|
|
if (!empty($decoded['errors'])) {
|
|
@file_put_contents(VV_CACHE_DIR . '/vv_api_debug.json', json_encode([
|
|
'ts' => time(),
|
|
'host' => $hostId,
|
|
'url' => $url,
|
|
'http_code' => $httpCode,
|
|
'errors' => $decoded['errors'],
|
|
'data' => $decoded['data'] ?? null,
|
|
], JSON_PRETTY_PRINT));
|
|
}
|
|
|
|
// data key present (even if null means query ran but returned nothing useful).
|
|
return array_key_exists('data', $decoded ?? []) ? $decoded['data'] : null;
|
|
}
|
|
|
|
// ── File-based API cache (/tmp/vv_cache — tmpfs, cleared on reboot) ───────────
|
|
|
|
// Read a cached payload. Returns null if missing or older than $maxAge seconds.
|
|
function vv_cache_read(string $key, int $maxAge = 90): ?array {
|
|
$f = VV_CACHE_DIR . '/' . $key . '.json';
|
|
if (!file_exists($f) || (time() - filemtime($f)) > $maxAge) return null;
|
|
$raw = file_get_contents($f);
|
|
return $raw ? (json_decode($raw, true) ?: null) : null;
|
|
}
|
|
|
|
// Write a payload atomically (tmp + rename) so readers never see a partial file.
|
|
function vv_cache_write(string $key, array $data): void {
|
|
if (!is_dir(VV_CACHE_DIR)) @mkdir(VV_CACHE_DIR, 0755, true);
|
|
$f = VV_CACHE_DIR . '/' . $key . '.json';
|
|
$tmp = $f . '.tmp';
|
|
file_put_contents($tmp, json_encode($data));
|
|
rename($tmp, $f);
|
|
}
|
|
|
|
// Drop a cached payload so the next read collects fresh. For use by endpoints that change the
|
|
// very state a cache describes: without it the UI polls a payload that cannot yet know about the
|
|
// action it just took, and the operator sees a container they stopped still running until the
|
|
// background writer next comes round.
|
|
//
|
|
// Best-effort by design. A cache that could not be removed is a stale read, which is what would
|
|
// have happened anyway — never a reason to fail the action that was actually requested.
|
|
function vv_cache_clear(string $key): void {
|
|
@unlink(VV_CACHE_DIR . '/' . $key . '.json');
|
|
}
|
|
|
|
// ── Shared utility functions (used across include/ and api/ files) ────────────
|
|
|
|
// Format seconds into "2d 3h 15m".
|
|
function vv_format_uptime(int $seconds): string {
|
|
$d = intdiv($seconds, 86400);
|
|
$h = intdiv($seconds % 86400, 3600);
|
|
$m = intdiv($seconds % 3600, 60);
|
|
return ($d ? "{$d}d " : '') . ($h ? "{$h}h " : '') . "{$m}m";
|
|
}
|
|
|
|
// ── Where a bash array ends ──────────────────────────────────────────────────────────────────
|
|
// Both halves of this had the same bug independently, which is why the rule now lives in one
|
|
// place: `[^)]*\)` — run to the first closing paren — is only the array's own close if no entry
|
|
// or comment contains one. HOST1_WATCHDOG_SCAN_IGNORE has carried
|
|
// `# broken test container (exit 127 — bad image)` for weeks, and the two sides failed differently
|
|
// against it:
|
|
//
|
|
// writing vv_conf_write_file() spliced the new body into the middle of that comment and left
|
|
// the real `)` stranded below as a stray token. `bash -n` caught it and vv_conf_edit()
|
|
// rolled the file back, so no conf was corrupted — but the caller got a bare `false`
|
|
// with an empty $rejected, and every future save of that list would have failed the
|
|
// same silent way.
|
|
// reading vv_parse_bash_array() stopped there and returned the entries above it, so the array
|
|
// was quietly short. Only the PHP layer was affected: bash sources the file itself and
|
|
// always saw every entry, so the watchdogs behaved correctly while the pages under-
|
|
// reported what they were configured with — the failure mode with no symptom.
|
|
//
|
|
// confform.php's own reader was always right, and this is its rule: a multi-line array closes on
|
|
// a `)` that starts its own line. Parens anywhere else are just text.
|
|
//
|
|
// CLOSE is non-capturing, for splicing a new body in. BODY captures — group 1 when the array is
|
|
// written on one line, group 2 when it spans several.
|
|
const VV_CONF_ARRAY_CLOSE = '(?:[^)\n]*\)|.*?\n[ \t]*\))';
|
|
const VV_CONF_ARRAY_BODY = '(?:([^)\n]*)\)|\n(.*?)^[ \t]*\)[ \t]*$)';
|
|
|
|
// Parse a scalar value from raw conf text. Matches KEY="value" or KEY=value.
|
|
// Identical logic was previously duplicated as vv_arr_scalar / vv_wd_scalar /
|
|
// vv_fb_scalar / vv_media_conf_scalar — all reduce to this one regex.
|
|
//
|
|
// Trailing comments are stripped, which the single-regex version did not do. It captured to the
|
|
// end of the line, so `FALLBACK_ENABLED=true # HOST2 back online` parsed as the whole string
|
|
// `true # HOST2 back online`. Numeric readings survived that — (int) and (float) stop at the
|
|
// first non-digit — which is why it went unnoticed: every threshold was right and every boolean
|
|
// was wrong. `=== 'true'` was false for any commented var, and worse, `!== 'false'` was TRUE for
|
|
// one, so a commented-out-to-off switch read as on. Roughly half of master.conf's toggles carry
|
|
// an inline comment.
|
|
//
|
|
// Quotes are honoured before that, because inside them a # is data, not a comment — a password
|
|
// or a colour would otherwise be truncated at the first hash. Unquoted, the comment must be
|
|
// introduced by whitespace, matching bash: FOO=#fff and FOO=bar#baz both assign literally, since
|
|
// # only opens a comment at the start of a word.
|
|
//
|
|
// The three regexes that did this are now one pass in vv_conf_unquote(), which handles escapes
|
|
// and adjacent quoted runs as well. See there for why that stopped being optional.
|
|
function vv_parse_conf_scalar(string $raw, string $key): string {
|
|
if (!preg_match('/^\s*' . preg_quote($key, '/') . '\s*=\s*(.*)$/m', $raw, $m)) return '';
|
|
return vv_conf_unquote(ltrim($m[1]));
|
|
}
|
|
|
|
// Read a bash array of plain strings out of a conf, skipping commented entries.
|
|
//
|
|
// Distinct from vv_parse_conf_array() in scheduler.php, which looks the same but keeps only
|
|
// entries ending in .sh — it exists to read job lists. Handing it a list of docker networks
|
|
// returns an empty array, silently, because none of them are scripts. This one makes no
|
|
// assumption about what the entries mean.
|
|
//
|
|
// The closing paren must be at the start of its own line, matching the shape conf_upgrade
|
|
// writes and the same anchor the scheduler parser uses — a value containing ')' would
|
|
// otherwise end the array early.
|
|
function vv_parse_conf_list(string $raw, string $key): array {
|
|
if (!preg_match('/^\s*' . preg_quote($key, '/') . '\s*=\s*\((.*?)^\s*\)/ms', $raw, $m)) {
|
|
return [];
|
|
}
|
|
$out = [];
|
|
foreach (explode("\n", $m[1]) as $line) {
|
|
if (preg_match('/^\s*#/', $line)) continue; // commented-out entry
|
|
if (!preg_match('/"([^"]*)"|\'([^\']*)\'/', $line, $e)) continue;
|
|
$val = trim($e[1] !== '' ? $e[1] : ($e[2] ?? ''));
|
|
if ($val !== '') $out[] = $val;
|
|
}
|
|
return $out;
|
|
}
|
|
|
|
// Unquote one bash word the way bash does, because the regexes this replaced did not and the
|
|
// conf is read by both. Three separate regexes each handled one quoting style in isolation and
|
|
// none of them handled an escape or two quoted runs in a row, so a value carrying a quote or a
|
|
// backslash parsed differently here than it did when a script sourced the same line. Nothing in
|
|
// the conf held one, which is the only reason it never showed.
|
|
//
|
|
// It matters now because a secret is written with its $ and ` escaped, so a password containing
|
|
// either is stored as "a\$b". The old regex captured everything between the quotes verbatim and
|
|
// would have shown a\$b — a backslash the operator never typed, in a field they are about to
|
|
// copy a credential out of, while every bash script that sourced the same line held a$b.
|
|
//
|
|
// Not a shell. No expansion of any kind, so ${DATA_DIR}/ai stays the literal text it is today —
|
|
// vv_conf_vars() owns resolving references, and doing it here would turn a display value into a
|
|
// different string than the one on disk.
|
|
// $end receives the offset where the word stopped, so a caller that also wants the trailing
|
|
// comment knows where the value ended without re-deriving it with a second regex that would
|
|
// disagree about quoting — which is exactly how the form came to show a # from inside a password
|
|
// as the start of a comment.
|
|
function vv_conf_unquote(string $v, ?int &$end = null): string {
|
|
$out = ''; $i = 0; $n = strlen($v);
|
|
while ($i < $n) {
|
|
$ch = $v[$i];
|
|
if ($ch === "'") {
|
|
// No escapes inside single quotes — the closing quote is the next one, always.
|
|
$j = strpos($v, "'", $i + 1);
|
|
if ($j === false) { $out .= substr($v, $i + 1); $i = $n; break; }
|
|
$out .= substr($v, $i + 1, $j - $i - 1);
|
|
$i = $j + 1;
|
|
} elseif ($ch === '"') {
|
|
$i++;
|
|
while ($i < $n && $v[$i] !== '"') {
|
|
// Only these four are escapes inside double quotes. A backslash before anything
|
|
// else is a literal backslash, which is why \d in a regex value survives.
|
|
if ($v[$i] === '\\' && $i + 1 < $n && strpos('\\"$`', $v[$i + 1]) !== false) {
|
|
$out .= $v[$i + 1]; $i += 2;
|
|
} else {
|
|
$out .= $v[$i]; $i++;
|
|
}
|
|
}
|
|
$i++;
|
|
} elseif ($ch === '\\' && $i + 1 < $n) {
|
|
$out .= $v[$i + 1]; $i += 2;
|
|
} elseif ($ch === ' ' || $ch === "\t" || $ch === "\r" || $ch === "\n") {
|
|
// End of the word. Everything after it is another word or a comment, and an
|
|
// unquoted conf value is one word by construction. \r is in the set because a conf
|
|
// saved with CRLF endings would otherwise carry one into every unquoted value.
|
|
break;
|
|
} else {
|
|
$out .= $ch; $i++;
|
|
}
|
|
}
|
|
// Clamped: an unterminated double quote runs $i one past the end.
|
|
$end = min($i, $n);
|
|
return $out;
|
|
}
|
|
|
|
// Parse a key=value state file (e.g. fallback_state.db, partnership_state.db).
|
|
// Returns ['key' => 'value', ...]. Lines without '=' are ignored.
|
|
function vv_parse_kv_db(string $text): array {
|
|
$out = [];
|
|
foreach (explode("\n", $text) as $line) {
|
|
$line = trim($line);
|
|
if ($line === '' || $line[0] === '#') continue;
|
|
[$k, $v] = array_pad(explode('=', $line, 2), 2, '');
|
|
if ($k !== '') $out[trim($k)] = trim($v);
|
|
}
|
|
return $out;
|
|
}
|
|
|
|
// All configured hosts from master.conf as ['host1' => 'hostname', ...].
|
|
// Canonical version — previously duplicated as vv_arr_known_hosts / vv_fb_known_hosts.
|
|
function vv_known_hosts(): array {
|
|
$vars = vv_conf_vars();
|
|
$hosts = [];
|
|
foreach ($vars as $k => $v) {
|
|
if (preg_match('/^HOST(\d+)$/', $k, $m) && $v !== '') {
|
|
$hosts['host' . $m[1]] = $v;
|
|
}
|
|
}
|
|
ksort($hosts);
|
|
return $hosts ?: ['host1' => 'HOST1'];
|
|
}
|
|
|
|
// Create (or overwrite) the Varaverk Unraid API key and write it into host conf.
|
|
// Returns ['ok'=>true,'key_preview'=>'...'] or ['ok'=>false,'error'=>'...'].
|
|
function vv_auto_create_api_key(string $hostId, string $confFile): array {
|
|
$script = SCRIPTS_DIR . '/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh';
|
|
if (!file_exists($script)) {
|
|
return ['ok' => false, 'error' => 'unraid_api_key_renew.sh not found'];
|
|
}
|
|
// set_time_limit() does not cover exec() time on Linux, so the bound has to be external —
|
|
// otherwise a stalled unraid-api call holds a php-fpm worker open indefinitely.
|
|
exec('timeout 120 bash ' . escapeshellarg($script) . ' 2>&1', $out, $rc);
|
|
if ($rc === 124) {
|
|
return ['ok' => false, 'error' => 'Key renewal timed out after 120s'];
|
|
}
|
|
if ($rc !== 0) {
|
|
$msg = implode(' ', array_filter(array_map('trim', $out)));
|
|
return ['ok' => false, 'error' => $msg ?: 'Script failed'];
|
|
}
|
|
$varName = strtoupper($hostId) . '_UNRAID_API_KEY';
|
|
$raw = vv_read_conf_raw($confFile);
|
|
preg_match('/^\s*' . preg_quote($varName, '/') . '\s*=\s*"([^"]+)"/m', $raw, $m);
|
|
$key = $m[1] ?? '';
|
|
return ['ok' => true, 'key_preview' => $key ? substr($key, 0, 8) . '...' . substr($key, -4) : 'registered'];
|
|
}
|
|
|
|
// Build a bash command that reads a state file from the REMOTE host's state directory.
|
|
// Reads the remote's varaverk.cfg to resolve their SCRIPTS_DIR (may differ from ours
|
|
// when the remote is in appdata mode). Falls back to the internal plugin path.
|
|
//
|
|
// The state directory is probed on the far side rather than assumed, because it moved:
|
|
// SCRIPTS_DIR/State_Files became DATA_DIR/state, and a partner may not have pulled that yet.
|
|
// This is the call that reads the partner's fallback_state.db, and a miss returns an empty
|
|
// string — which the callers cannot distinguish from "partner is in NORMAL state". Reading the
|
|
// wrong directory would therefore not look like an error, it would look like an answer. Probing
|
|
// also means the two hosts can be upgraded in either order.
|
|
function vv_remote_state_cmd(string $filename): string {
|
|
$fn = basename($filename);
|
|
return 'sd=$(grep -m1 SCRIPTS_DIR= /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null'
|
|
. ' | cut -d\'"\' -f2); sd="${sd:-/boot/config/plugins/varaverk}"; '
|
|
. 'sf="$sd/data/state"; [ -d "$sf" ] || sf="$sd/State_Files"; '
|
|
. 'cat "$sf/' . $fn . '" 2>/dev/null';
|
|
}
|
|
|
|
// Local LAN IP via routing table — static-cached per request.
|
|
// Previously duplicated in include/docker_folders.php and inline in include/docker.php.
|
|
function vv_local_ip(): string {
|
|
static $ip = null;
|
|
if ($ip !== null) return $ip;
|
|
$ip = trim(shell_exec("ip route get 8.8.8.8 2>/dev/null | awk '/src/{for(i=1;i<=NF;i++)if(\$i==\"src\")print \$(i+1)}'") ?? '');
|
|
return $ip;
|
|
}
|