Varaverk/Plugin/unraid/include/monitor.php

<?php
require_once __DIR__ . '/common.php';

// Monitor-page-specific helpers — partner state, fallback state, watchdog summary, scripts status.

function vv_partner_state(): array {
    $vars   = vv_conf_vars();
    $myName = trim(shell_exec('hostname -s') ?: '');

    // Parse Tailscale peer status once
    $tsData  = json_decode(shell_exec('tailscale status --json 2>/dev/null') ?: '{}', true) ?? [];
    $tsPeers = [];
    foreach ($tsData['Peer'] ?? [] as $peer) {
        // DNSName is "hostname.tailnet.ts.net." — take the first label (full, not truncated)
        $dns = $peer['DNSName'] ?? '';
        $h   = $dns ? strtolower(explode('.', $dns)[0]) : strtolower($peer['HostName'] ?? '');
        if ($h) $tsPeers[$h] = (bool)($peer['Online'] ?? false);
    }

    $hostIds = array_filter(array_keys($vars), fn($k) => preg_match('/^HOST\d+$/', $k) && ($vars[$k] ?? '') !== '');
    sort($hostIds);

    $hosts = [];
    foreach ($hostIds as $id) {
        $hostname = $vars[$id] ?? '';
        if (!$hostname) continue;
        $isMe    = strcasecmp($hostname, $myName) === 0;
        $isOwner = strcasecmp($id, $vars['PARTNERSHIP_OWNER_HOST'] ?? '') === 0;
        $online  = $isMe ? true : ($tsPeers[strtolower($hostname)] ?? null);
        $hosts[] = [
            'id'       => $id,
            'hostname' => $hostname,
            'owner'    => $vars[$id . '_OWNER'] ?? '',
            'is_me'    => $isMe,
            'is_owner' => $isOwner,
            'online'   => $online,
        ];
    }

    return [
        'enabled'    => ($vars['PARTNERSHIP_ENABLED'] ?? 'false') === 'true',
        'owner_host' => $vars['PARTNERSHIP_OWNER_HOST'] ?? '',
        'sync_min'   => (int)($vars['PARTNERSHIP_SYNC_INTERVAL'] ?? 15),
        'hosts'      => $hosts,
    ];
}

function vv_fallback_state(): array {
    // State file written by fallback.sh
    $stateFile = '/tmp/fallback_state.db';
    if (!file_exists($stateFile)) return ['state' => 'UNKNOWN'];
    $raw = [];
    foreach (file($stateFile) ?: [] as $line) {
        [$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
        $raw[trim($k)] = trim($v);
    }
    return [
        'state'          => $raw['state']          ?? 'UNKNOWN',
        'failover_start' => $raw['failover_start']  ?? '0',
        'tier2_started'  => $raw['tier2_started']   ?? 'false',
        'tier3_started'  => $raw['tier3_started']   ?? 'false',
        'tier4_started'  => $raw['tier4_started']   ?? 'false',
    ];
}

function vv_fallback_active(): array {
    $vars   = vv_conf_vars();
    $myName = trim(shell_exec('hostname -s') ?: '');

    // Identify which HOST id we are
    $allHostIds = array_filter(array_keys($vars), fn($k) => preg_match('/^HOST\d+$/', $k) && ($vars[$k] ?? '') !== '');
    sort($allHostIds);
    $myId = null;
    foreach ($allHostIds as $id) {
        if (strcasecmp($vars[$id] ?? '', $myName) === 0) { $myId = $id; break; }
    }
    if (!$myId) return [];

    // Running containers: name → image
    $running = [];
    $psOut = shell_exec("docker ps --format '{\"n\":\"{{.Names}}\",\"i\":\"{{.Image}}\"}' 2>/dev/null") ?: '';
    foreach (explode("\n", trim($psOut)) as $line) {
        $c = json_decode($line, true);
        if ($c) $running[strtolower($c['n'])] = $c['i'];
    }

    // Parse FALLBACK arrays from this host's conf
    $confFile = strtolower($myId) . '.conf';
    $rawConf  = vv_read_conf_raw($confFile);

    $result = [];
    foreach ($allHostIds as $covered) {
        if ($covered === $myId) continue;
        $coveredHostname = $vars[$covered] ?? '';
        if (!$coveredHostname) continue;

        $names = [];
        for ($tier = 1; $tier <= 4; $tier++)
            $names = array_merge($names, vv_parse_bash_array($rawConf, "FALLBACK_{$myId}_COVERS_{$covered}_TIER{$tier}"));

        $active = [];
        foreach ($names as $name) {
            if (isset($running[strtolower($name)]))
                $active[] = ['name' => $name, 'image' => $running[strtolower($name)]];
        }

        if ($active) $result[] = ['host_id' => $covered, 'hostname' => $coveredHostname, 'containers' => $active];
    }

    return $result;
}

function vv_watchdog_summary(): array {
    $parseKv = function(string $raw): array {
        $out = [];
        foreach (explode("\n", $raw) as $line) {
            $line = trim($line);
            if (str_contains($line, ':'))      { [$k, $v] = explode(':', $line, 2); $out[trim($k)] = trim($v); }
            elseif (str_contains($line, '='))  { [$k, $v] = explode('=', $line, 2); $out[trim($k)] = trim($v, '"\''); }
        }
        return $out;
    };

    $dock = $parseKv(@file_get_contents('/tmp/container_watchdog_state.db')  ?: '');
    $rw   = $parseKv(@file_get_contents('/tmp/resource_watchdog_state.db')   ?: '');

    $ctrStrikes = [];
    foreach ($dock as $k => $v) {
        if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0)
            $ctrStrikes[$k] = (int)$v;
    }

    // Recent restarts (24 h)
    $restartLog = '/mnt/user/appdata/unraid_scripts/data/container_restart_history.db';
    $restartRaw = @file_get_contents($restartLog) ?: '';
    $cutoff     = time() - 86400;
    $restarts   = [];
    foreach (explode("\n", trim($restartRaw)) as $line) {
        if (!$line || !str_contains($line, '|')) continue;
        [$name, $ts] = explode('|', $line, 2);
        if ((int)$ts >= $cutoff) $restarts[] = ['name' => trim($name), 'ts' => (int)$ts];
    }
    usort($restarts, fn($a, $b) => $b['ts'] - $a['ts']);

    // Reboots (12 h)
    $rebootRaw = @file_get_contents('/boot/config/system_watchdog_reboots.db') ?: '';
    $rbootCutoff = time() - 43200;
    $reboots = 0;
    foreach (explode("\n", trim($rebootRaw)) as $line) {
        if ((int)trim($line) >= $rbootCutoff) $reboots++;
    }

    $rwLevel       = (int)($rw['rm_action_level']  ?? 0);
    $daemonStrikes = (int)($dock['daemon_strikes']  ?? 0);
    $oomCount      = (int)trim(@file_get_contents('/tmp/system_watchdog_oom.db') ?: '0');

    // ── Stability watchdog strikes (/tmp/system_watchdog_state.db) ───────────
    $stabRaw = @file_get_contents('/tmp/system_watchdog_state.db') ?: '';
    $stabStrikes = [];
    foreach (explode("\n", $stabRaw) as $line) {
        $line = trim($line);
        if (!$line || !str_contains($line, ':')) continue;
        [$k, $v] = explode(':', $line, 2);
        $count = (int)trim($v);
        if ($count > 0) $stabStrikes[trim($k)] = $count;
    }

    // ── Storage watchdog strikes (/tmp/storage_watchdog_state.db) ────────────
    $storRaw = @file_get_contents('/tmp/storage_watchdog_state.db') ?: '';
    $growthStrikes = []; $logStrikes = [];
    foreach (explode("\n", $storRaw) as $line) {
        $line = trim($line);
        if (!$line || !str_contains($line, ':')) continue;
        [$k, $v] = explode(':', $line, 2);
        $count = (int)trim($v);
        if ($count <= 0) continue;
        $key = trim($k);
        if (str_starts_with($key, 'appdata_growth_'))
            $growthStrikes[substr($key, strlen('appdata_growth_'))] = $count;
        elseif (str_starts_with($key, 'appdata_log_'))
            $logStrikes[substr($key, strlen('appdata_log_'))] = $count;
    }

    // ── Network watchdog NPM strikes (/tmp/network_watchdog_state.db) ────────
    $netRaw = @file_get_contents('/tmp/network_watchdog_state.db') ?: '';
    $npmStrikes = 0;
    foreach (explode("\n", $netRaw) as $line) {
        $line = trim($line);
        if (str_starts_with($line, 'npm:')) $npmStrikes = (int)trim(substr($line, 4));
    }

    // ── Stability live stats ──────────────────────────────────────────────────
    $dfPct = function(string $path): int {
        $out = shell_exec("df " . escapeshellarg($path) . " --output=pcent 2>/dev/null | tail -1") ?: '';
        return (int)trim(str_replace('%', '', $out));
    };

    $memRaw   = @file_get_contents('/proc/meminfo') ?: '';
    $memAvail = 0;
    if (preg_match('/^MemAvailable:\s+(\d+)/m', $memRaw, $m)) $memAvail = (int)$m[1];

    $loadRaw = @file_get_contents('/proc/loadavg') ?: '0';
    $load1   = (float)explode(' ', trim($loadRaw))[0];

    $cpuTemp = null;
    $sensorsOut = shell_exec("sensors 2>/dev/null | grep -E 'Core 0|Package id 0|Tdie|Tctl|CPU Temp' | grep -oE '[0-9]+\\.[0-9]+' | sort -n | tail -1") ?: '';
    if ($sensorsOut && is_numeric(trim($sensorsOut))) $cpuTemp = (int)round((float)trim($sensorsOut));

    $zombies = (int)trim(shell_exec("ps -eo stat 2>/dev/null | grep -c '^Z'") ?: '0');

    $nic      = trim(shell_exec("ip route show default 2>/dev/null | awk 'NR==1{print \$5}'") ?: 'eth0') ?: 'eth0';
    $nicState = trim(@file_get_contents("/sys/class/net/$nic/operstate") ?: 'unknown');
    $sshdOk   = (int)trim(shell_exec('pgrep -c sshd 2>/dev/null') ?: '0') > 0;

    $healthy = empty($ctrStrikes) && empty($stabStrikes) && empty($growthStrikes) && empty($logStrikes)
        && $rwLevel === 0 && $daemonStrikes === 0 && $oomCount === 0 && $reboots === 0
        && $npmStrikes === 0 && $nicState === 'up' && $sshdOk;

    return [
        'healthy'        => $healthy,
        'ctr_strikes'    => $ctrStrikes,
        'rw_level'       => $rwLevel,
        'rw_paused'      => array_values(array_filter(explode(',', $rw['rm_paused_containers'] ?? ''))),
        'rw_stopped'     => array_values(array_filter(explode(',', $rw['rm_stopped_containers'] ?? ''))),
        'daemon_strikes' => $daemonStrikes,
        'oom_count'      => $oomCount,
        'reboots_12h'    => $reboots,
        'restarts_24h'   => array_slice($restarts, 0, 6),
        'restart_count'  => count($restarts),
        'stability'      => [
            'rootfs_pct'  => $dfPct('/'),
            'log_pct'     => $dfPct('/var/log'),
            'tmp_pct'     => $dfPct('/tmp'),
            'ram_free_gb' => round($memAvail / 1048576, 1),
            'load_1min'   => $load1,
            'cpu_temp'    => $cpuTemp,
            'zombies'     => $zombies,
            'nic'         => $nic,
            'nic_state'   => $nicState,
            'sshd_ok'     => $sshdOk,
            'strikes'     => $stabStrikes,
        ],
        'storage_wd'     => [
            'growth_strikes' => $growthStrikes,
            'log_strikes'    => $logStrikes,
        ],
        'network_wd'     => [
            'npm_strikes' => $npmStrikes,
        ],
    ];
}

function vv_scripts_status(): array {
    $logDir    = LOG_DIR;
    $statFiles = array_merge(
        glob("$logDir/*.json")   ?: [],
        glob("$logDir/*/*.json") ?: []
    );

    $scripts = [];
    foreach ($statFiles as $statFile) {
        $stat   = json_decode(@file_get_contents($statFile) ?: '{}', true) ?: [];
        $status = $stat['status'] ?? 'unknown';

        // Stale running — PID gone (crash or reboot with no cleanup)
        if ($status === 'running' && !empty($stat['pid'])) {
            if (!file_exists("/proc/{$stat['pid']}")) $status = 'error';
        }

        $id   = $stat['id'] ?? basename($statFile, '.json');
        $name = basename(preg_replace('/\.sh$/', '', $id));
        $ts   = (int)($stat['end'] ?? $stat['start'] ?? @filemtime($statFile) ?: 0);

        $scripts[] = [
            'name'     => $name,
            'last_ts'  => $ts,
            'status'   => $status,
            'running'  => $status === 'running',
            'exit'     => $stat['exit'] ?? null,
            'duration' => isset($stat['start'], $stat['end'])
                ? (int)$stat['end'] - (int)$stat['start'] : null,
        ];
    }

    usort($scripts, fn($a, $b) => ($b['last_ts'] ?? 0) <=> ($a['last_ts'] ?? 0));
    $scripts = array_slice($scripts, 0, 12);

    return [
        'scripts'       => $scripts,
        'running_count' => count(array_filter($scripts, fn($s) => $s['status'] === 'running')),
        'ok_count'      => count(array_filter($scripts, fn($s) => $s['status'] === 'ok')),
        'warn_count'    => count(array_filter($scripts, fn($s) => $s['status'] === 'warn')),
        'error_count'   => count(array_filter($scripts, fn($s) => $s['status'] === 'error')),
    ];
}