($v['PARTNERSHIP_ENABLED'] ?? 'false') === 'true', 'owner_host' => $v['PARTNERSHIP_OWNER_HOST'] ?? '', 'sync_min' => (int)($v['PARTNERSHIP_SYNC_INTERVAL'] ?? 15), 'grace_hours' => (int)($v['PARTNERSHIP_GRACE_HOURS'] ?? 6), 'offline_threshold' => (int)($v['PARTNERSHIP_OFFLINE_THRESHOLD'] ?? 30), 'offline_days' => $offlineDays, 'remove_tailscale' => ($v['PARTNERSHIP_REMOVE_TAILSCALE'] ?? 'true') === 'true', 'folderview3' => ($v['PARTNERSHIP_FOLDERVIEW3'] ?? 'false') === 'true', 'tailscale_configured' => !empty($v['TAILSCALE_API_KEY']) && !empty($v['TAILSCALE_TAILNET']), 'transfer_confirm' => $v['PARTNERSHIP_TRANSFER_CONFIRM'] ?? 'i-understand-this-transfers-ownership', ]; } // ── Mirror sync health — the partnership's actual job (rsync orchestrators) ────── function vv_pt_sync(): array { $jobs = [ 'critical' => 'Orchestrators/critical_sync_maintenance', 'daily' => 'Orchestrators/daily_sync_maintenance', 'weekly' => 'Orchestrators/weekly_sync_maintenance', ]; $out = ['jobs' => []]; foreach ($jobs as $key => $base) { $statFile = LOG_DIR . '/' . $base . '.json'; $s = file_exists($statFile) ? json_decode(@file_get_contents($statFile), true) : null; $out['jobs'][$key] = is_array($s) ? [ 'status' => $s['status'] ?? 'unknown', 'start' => isset($s['start']) ? (int)$s['start'] : null, 'end' => isset($s['end']) ? (int)$s['end'] : null, ] : null; } $v = vv_conf_vars(); // Rsync gate flags (master.conf) — global Tier 1 + per-tier Tier 2. $out['gates'] = [ 'global' => ['var' => 'RSYNC_ENABLED', 'on' => ($v['RSYNC_ENABLED'] ?? 'true') === 'true'], 'critical' => ['var' => 'CRITICAL_RSYNC_ENABLED', 'on' => ($v['CRITICAL_RSYNC_ENABLED'] ?? 'true') === 'true'], 'daily' => ['var' => 'DAILY_RSYNC_ENABLED', 'on' => ($v['DAILY_RSYNC_ENABLED'] ?? 'true') === 'true'], 'weekly' => ['var' => 'WEEKLY_RSYNC_ENABLED', 'on' => ($v['WEEKLY_RSYNC_ENABLED'] ?? 'true') === 'true'], ]; // Back-compat keys still used by the warning line. $out['rsync_enabled'] = $out['gates']['global']['on']; $out['critical_enabled'] = $out['gates']['critical']['on']; return $out; } // ── State file parser ───────────────────────────────────────────────────────── function vv_pt_read_db(string $path): array { if (!file_exists($path)) return []; $out = []; foreach (file($path, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) as $line) { [$k, $v] = array_pad(explode('=', trim($line), 2), 2, ''); if ($k) $out[trim($k)] = trim($v, '"\''); } return $out; } // ── Tailscale peers ─────────────────────────────────────────────────────────── function vv_pt_ts_peers(): array { $raw = shell_exec('tailscale status --json 2>/dev/null') ?: '{}'; $data = json_decode($raw, true) ?: []; $peers = []; // Self $self = $data['Self'] ?? []; $selfLabel = strtolower(explode('.', $self['DNSName'] ?? '')[0]); if ($selfLabel) { $peers[$selfLabel] = [ 'online' => true, 'active' => true, 'ip' => $self['TailscaleIPs'][0] ?? null, ]; } // Peers foreach ($data['Peer'] ?? [] as $peer) { $label = strtolower(explode('.', $peer['DNSName'] ?? '')[0]); if (!$label) continue; $peers[$label] = [ 'online' => (bool)($peer['Online'] ?? false), 'active' => (bool)($peer['Active'] ?? false), 'ip' => $peer['TailscaleIPs'][0] ?? null, ]; } return $peers; } // ── SSH helper — run a single command on a remote host ──────────────────────── function vv_pt_ssh(string $ip, string $sshKey, string $cmd, int $timeout = 4): string { if (!$ip || !$sshKey || !file_exists($sshKey)) return ''; $full = sprintf( 'ssh -i %s -o ConnectTimeout=%d -o StrictHostKeyChecking=no -o BatchMode=yes root@%s %s 2>/dev/null', escapeshellarg($sshKey), $timeout, escapeshellarg($ip), escapeshellarg($cmd) ); return shell_exec($full) ?: ''; } // ── System info ─────────────────────────────────────────────────────────────── // vv_system_info() (common.php) provides version, load_avg, array_state. // /proc/uptime is the reliable uptime source (API uptime is an ISO date string, not seconds). // vv_docker_containers() (common.php) provides the running container list. function vv_pt_local_system(): array { $info = vv_system_info(); $uptimeSec = file_exists('/proc/uptime') ? (int)explode(' ', file_get_contents('/proc/uptime'))[0] : 0; return [ 'unraid_version' => $info['version'] ?? '', 'uptime_sec' => $uptimeSec, 'load_avg' => isset($info['load_avg']) ? $info['load_avg'][0] : null, 'containers' => count(vv_docker_containers()), ]; } // SSH fallback for remote nodes — version/uptime/load/containers in one call. // API stats from vv_remote_hosts_stats() take priority when available; SSH fills gaps. function vv_pt_remote_system(string $ip, string $sshKey): array { $out = vv_pt_ssh($ip, $sshKey, 'printf "%s\nUPTIME:%s\nLOAD:%s\nCONTAINERS:%s\n" ' . '"$(cat /etc/unraid-version 2>/dev/null)" ' . '"$(cat /proc/uptime 2>/dev/null)" ' . '"$(awk \'{print $1}\' /proc/loadavg 2>/dev/null)" ' . '"$(docker ps -q 2>/dev/null | wc -l)"'); $ver = ''; preg_match('/VERSION="([^"]+)"/', $out, $m); if ($m) $ver = $m[1]; $uptime = 0; if (preg_match('/UPTIME:([\d.]+)/', $out, $m)) $uptime = (int)$m[1]; $load = null; if (preg_match('/LOAD:([\d.]+)/', $out, $m)) $load = round((float)$m[1], 2); $containers = null; if (preg_match('/CONTAINERS:(\d+)/', $out, $m)) $containers = (int)$m[1]; return [ 'unraid_version' => $ver, 'uptime_sec' => $uptime, 'load_avg' => $load, 'containers' => $containers, ]; } // ── Per-node data ───────────────────────────────────────────────────────────── function vv_pt_nodes(): array { $currentHost = vv_detect_host(); $hosts = vv_arr_known_hosts(); // ['host1' => 'hostname', ...] $vars = vv_conf_vars(); $tsPeers = vv_pt_ts_peers(); $ownerSlot = strtolower($vars['PARTNERSHIP_OWNER_HOST'] ?? ''); $setupDb = vv_setup_state_read(); // Remote host stats (API + 30s /tmp cache) — includes version, uptime, cpu/ram/array/temp/vms $remoteStats = vv_remote_hosts_stats(); // SSH key for this host $myId = strtoupper($currentHost); $myRaw = vv_read_conf_raw($currentHost . '.conf'); $mySshKey = vv_arr_scalar($myRaw, $myId . '_SSH_KEY'); $nodes = []; foreach ($hosts as $slot => $hostname) { $isMe = ($slot === $currentHost || $currentHost === 'unknown'); $isOwner = (strtolower($ownerSlot) === $slot); // Tailscale $tsLabel = strtolower($hostname); $ts = $tsPeers[$tsLabel] ?? ['online' => null, 'active' => false, 'ip' => null]; // Fallback state $fbState = 'UNKNOWN'; $fbPath = '/boot/config/fallback_state.db'; if ($isMe) { $fb = vv_pt_read_db($fbPath); $fbState = $fb['state'] ?? 'UNKNOWN'; } elseif ($ts['online'] && $ts['ip'] && $mySshKey) { $out = vv_pt_ssh($ts['ip'], $mySshKey, 'cat /boot/config/fallback_state.db 2>/dev/null'); if ($out) { $fb = []; foreach (explode("\n", $out) as $line) { [$k, $v] = array_pad(explode('=', trim($line), 2), 2, ''); if ($k) $fb[trim($k)] = trim($v, '"\''); } $fbState = $fb['state'] ?? 'UNKNOWN'; } } // Partnership DB — local only (each server writes its own) $dbPath = "/boot/config/partnership_{$hostname}.db"; $ptDb = vv_pt_read_db($dbPath); // System info $system = $isMe ? vv_pt_local_system() : ($ts['online'] && $ts['ip'] && $mySshKey ? vv_pt_remote_system($ts['ip'], $mySshKey) : []); // Onboard phase from setup.db — null=self, 0=not started, 1=SSH+conf done, 2=fully onboarded $nodeIdUpper = strtoupper($slot); $onboardPhase = $isMe ? null : (($setupDb[$nodeIdUpper . '_PHASE2_DONE'] ?? '') === 'true' ? 2 : (($setupDb[$nodeIdUpper . '_PHASE1_DONE'] ?? '') === 'true' ? 1 : 0)); // key_ready: local key generated but not yet installed on HOST2 (SSH pending manual step) $keyReady = !$isMe && ($setupDb[$nodeIdUpper . '_KEY_READY'] ?? '') === 'true'; // For self: local setup complete flag (set by partnership_manager --onboard --local-only) $localDone = $isMe && ($setupDb[$nodeIdUpper . '_LOCAL_DONE'] ?? '') === 'true'; // Unraid API key status — checks Unraid's key store directly so deletions are reflected. $apiKeySet = false; $apiKeyPreview = ''; if ($isMe) { $apiOut = shell_exec('/usr/local/sbin/unraid-api apikey --name "Varaverk" --json /dev/null'); $apiData = json_decode(trim($apiOut ?? ''), true); if (is_array($apiData) && !empty($apiData['key'])) { $apiKeySet = true; $apiKeyPreview = substr($apiData['key'], 0, 8) . '...' . substr($apiData['key'], -4); } } // Live metrics: local uses vv_api_data() (cached); remote uses vv_remote_hosts_stats() (30s cache) if ($isMe) { $metrics = array_merge( vv_api_node_metrics(vv_api_data()), array_filter([ 'load_avg' => $system['load_avg'] ?? null, 'containers' => $system['containers'] ?? null, ], fn($v) => $v !== null) ); } else { $rStat = $remoteStats[$nodeIdUpper] ?? []; // Merge API metrics from remote stats with SSH extras (load, containers) $metrics = array_filter([ 'cpu_pct' => $rStat['cpu_pct'] ?? null, 'ram_used_gb' => $rStat['ram_used_gb'] ?? null, 'ram_total_gb' => $rStat['ram_total_gb'] ?? null, 'array_used_tb' => $rStat['array_used_tb'] ?? null, 'array_total_tb' => $rStat['array_total_tb'] ?? null, 'max_disk_temp' => $rStat['max_disk_temp'] ?? null, 'vm_count' => $rStat['vm_count'] ?? null, 'load_avg' => $system['load_avg'] ?? null, 'containers' => $system['containers'] ?? null, ], fn($v) => $v !== null); // Fill version/uptime from API stats if SSH didn't provide them if (empty($system['unraid_version']) && !empty($rStat['version'])) { $system['unraid_version'] = $rStat['version']; } if (empty($system['uptime_sec']) && !empty($rStat['uptime_sec'])) { $system['uptime_sec'] = $rStat['uptime_sec']; } } $nodes[] = [ 'slot' => $slot, 'id' => $nodeIdUpper, 'hostname' => $hostname, 'is_me' => $isMe, 'is_owner' => $isOwner, 'ts_online' => $ts['online'], 'ts_active' => $ts['active'], 'ts_ip' => $ts['ip'], 'fallback' => $fbState, 'partnership' => $ptDb, 'system' => $system, 'onboard_phase' => $onboardPhase, 'key_ready' => $keyReady, 'local_done' => $localDone, 'api_key_set' => $apiKeySet, 'api_key_preview' => $apiKeyPreview, 'metrics' => $metrics, ]; } return $nodes; } // ── Connectivity test — SSH echo with round-trip timing ───────────────────────── function vv_pt_ping(string $slot): array { $slot = strtolower($slot); $vars = vv_conf_vars(); $hostname = $vars[strtoupper($slot)] ?? ''; if (!$hostname) return ['ok' => false, 'error' => 'Unknown host slot']; $currentHost = vv_detect_host(); $myRaw = vv_read_conf_raw($currentHost . '.conf'); $sshKey = vv_arr_scalar($myRaw, strtoupper($currentHost) . '_SSH_KEY'); if (!$sshKey || !file_exists($sshKey)) { return ['ok' => false, 'error' => 'No SSH key configured on this host']; } $ip = vv_resolve_tailscale_ip($hostname); if (!$ip) return ['ok' => false, 'error' => "Cannot resolve Tailscale IP for $hostname"]; $t0 = microtime(true); $out = vv_pt_ssh($ip, $sshKey, 'echo ok', 8); $ms = (int)round((microtime(true) - $t0) * 1000); if (trim($out) === 'ok') { return ['ok' => true, 'latency_ms' => $ms, 'host' => $hostname, 'ip' => $ip]; } return ['ok' => false, 'error' => "SSH to $hostname ($ip) failed or timed out", 'host' => $hostname]; } // ── Entry point ─────────────────────────────────────────────────────────────── function vv_partnership_all(): array { return [ 'config' => vv_pt_config(), 'nodes' => vv_pt_nodes(), 'sync' => vv_pt_sync(), 'ts' => time(), ]; }