<?php
require_once __DIR__ . '/common.php';

// ═══════════════════════════════════════════════════════════════════════════════════════════════
// PURPOSE
//   Monitor-page roll-ups that are not raw system metrics: partner reachability, fallback
//   state, the single-glance watchdog health summary, script run status, and rsync progress.
//   Raw hardware numbers come from common.php; this file answers "is anything wrong".
//
// DESIGN PRINCIPLES
//   One boolean has to be trustworthy.
//       vv_watchdog_summary() reduces every watchdog's state to 'healthy'. It is the only
//       thing most people look at, so it is conjunctive — healthy requires every strike set
//       empty, every level zero, the NIC up and sshd alive. Any doubt resolves to not-healthy.
//
//   State paths derive from STATE_DIR, never hardcoded.
//       The watchdogs write under STATE_DIR, which follows SCRIPTS_DIR through a storage-mode
//       migration. Hardcoding an absolute path here silently decouples the page from the
//       scripts — see OPERATIONAL SAFEGUARDS.
//
//   Summarises; does not re-derive.
//       Strike counts come from the files the watchdogs wrote. This file never recomputes
//       whether a container is unhealthy — that decision belongs to the watchdog that owns it.
//
// OPERATIONAL SAFEGUARDS
//   Missing state must not read as healthy — and once did.
//       Six state files were read from /tmp while the watchdogs write to STATE_DIR. Every
//       read returned empty, every strike set came back clear, and 'healthy' was therefore
//       always true: a permanent false all-clear on the page whose whole job is raising the
//       alarm. Fixed 2026-08-02. If a strike set ever looks suspiciously empty, verify the
//       path against where the watchdog actually writes before trusting it.
//
//   Live stability probes are cheap and time-boxed.
//       df, sensors, pgrep and ps run per render, so each is a single command with stderr
//       discarded and a scalar result. Nothing here iterates over containers or disks.
//
//   Shell arguments are escaped.
//       Paths passed to df go through escapeshellarg(); the NIC name is read from sysfs
//       rather than interpolated from user input.
//
//   Read-only. Reports on watchdogs, fallback and scripts; never starts, stops, or clears any
//   of them.
//
// EXPORTS
//   vv_partner_state()      partner reachability and identity
//   vv_fallback_state()     current fallback state for this host
//   vv_fallback_active()    whether this host is currently covering, and what
//   vv_watchdog_summary()   the health roll-up described above
//   vv_scripts_status()     last-run status per scheduled script
//   vv_rsync_status()       current/last rsync progress
//
// CONFIGURATION
//   STATE_DIR   fallback_state.db, container/resource/system/storage/network watchdog state,
//               system_watchdog_oom.db, system_watchdog_reboots.db
//   DATA_DIR    container_restart_history.db
// ═══════════════════════════════════════════════════════════════════════════════════════════════

function vv_partner_state(): array {
    $vars     = vv_conf_vars();
    $myHostId = strtoupper(vv_detect_host());

    // Parse Tailscale peer status once
    $tsData  = json_decode(shell_exec('tailscale status --json 2>/dev/null') ?: '{}', true) ?? [];
    $tsPeers = [];
    foreach ($tsData['Peer'] ?? [] as $peer) {
        // DNSName is "hostname.tailnet.ts.net." — take the first label (full, not truncated)
        $dns = $peer['DNSName'] ?? '';
        $h   = $dns ? strtolower(explode('.', $dns)[0]) : strtolower($peer['HostName'] ?? '');
        if ($h) $tsPeers[$h] = (bool)($peer['Online'] ?? false);
    }

    $hostIds = array_filter(array_keys($vars), fn($k) => preg_match('/^HOST\d+$/', $k) && ($vars[$k] ?? '') !== '');
    sort($hostIds);

    $setupDb = vv_setup_state_read();

    $hosts = [];
    foreach ($hostIds as $id) {
        $hostname = $vars[$id] ?? '';
        if (!$hostname) continue;
        $isMe    = ($id === $myHostId);
        $isOwner = strcasecmp($id, $vars['PARTNERSHIP_OWNER_HOST'] ?? '') === 0;
        $online  = $isMe ? true : ($tsPeers[strtolower($hostname)] ?? null);
        $onboardPhase = $isMe ? null
            : (($setupDb[$id . '_PHASE2_DONE'] ?? '') === 'true' ? 2
              : (($setupDb[$id . '_PHASE1_DONE'] ?? '') === 'true' ? 1 : 0));
        $hosts[] = [
            'id'            => $id,
            'hostname'      => $hostname,
            'owner'         => $vars[$id . '_OWNER'] ?? '',
            'is_me'         => $isMe,
            'is_owner'      => $isOwner,
            'online'        => $online,
            'onboard_phase' => $onboardPhase,
        ];
    }

    return [
        'enabled'    => ($vars['PARTNERSHIP_ENABLED'] ?? 'false') === 'true',
        'owner_host' => $vars['PARTNERSHIP_OWNER_HOST'] ?? '',
        'sync_min'   => (int)($vars['PARTNERSHIP_SYNC_INTERVAL'] ?? 15),
        'hosts'      => $hosts,
    ];
}

function vv_fallback_state(): array {
    $vars      = vv_conf_vars();
    $enabled   = ($vars['FALLBACK_ENABLED']           ?? 'false') === 'true';
    $interval  = (int)($vars['FALLBACK_CHECK_INTERVAL']    ?? 30);
    $reqStrikes = (int)($vars['FALLBACK_HANDBACK_STRIKES'] ?? 3);
    $suspendAfter = (int)($vars['FALLBACK_PARTNERSHIP_SUSPEND_AFTER'] ?? 120);

    $stateFile = STATE_DIR . '/fallback_state.db';
    if (!file_exists($stateFile)) {
        return ['state' => 'UNKNOWN', 'enabled' => $enabled, 'check_interval' => $interval,
                'handback_strikes' => 0, 'handback_strikes_required' => $reqStrikes,
                'partnership_suspended' => false,
                'partner_lost_at' => 0, 'partnership_suspend_after' => $suspendAfter];
    }
    $raw = [];
    foreach (file($stateFile) ?: [] as $line) {
        [$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
        $raw[trim($k)] = trim($v);
    }
    return [
        'state'                      => $raw['state']               ?? 'UNKNOWN',
        'failover_start'             => $raw['failover_start']       ?? '0',
        'tier2_started'              => $raw['tier2_started']        ?? 'false',
        'tier3_started'              => $raw['tier3_started']        ?? 'false',
        'tier4_started'              => $raw['tier4_started']        ?? 'false',
        'handback_strikes'           => (int)($raw['handback_strikes']    ?? 0),
        'partnership_suspended'      => ($raw['partnership_suspended']    ?? 'false') === 'true',
        'partner_lost_at'            => (int)($raw['partner_lost_at']     ?? 0),
        'enabled'                    => $enabled,
        'check_interval'             => $interval,
        'handback_strikes_required'  => $reqStrikes,
        'partnership_suspend_after'  => $suspendAfter,
    ];
}

function vv_fallback_active(): array {
    $vars = vv_conf_vars();

    // Identify which HOST id we are
    $allHostIds = array_filter(array_keys($vars), fn($k) => preg_match('/^HOST\d+$/', $k) && ($vars[$k] ?? '') !== '');
    sort($allHostIds);
    $myId = strtoupper(vv_detect_host());
    if ($myId === 'UNKNOWN' || !in_array($myId, $allHostIds, true)) return [];

    // Running containers: name → image
    $running = [];
    $psOut = shell_exec("docker ps --format '{\"n\":\"{{.Names}}\",\"i\":\"{{.Image}}\"}' 2>/dev/null") ?: '';
    foreach (explode("\n", trim($psOut)) as $line) {
        $c = json_decode($line, true);
        if ($c) $running[strtolower($c['n'])] = $c['i'];
    }

    $result = [];
    foreach ($allHostIds as $covered) {
        if ($covered === $myId) continue;
        $coveredHostname = $vars[$covered] ?? '';
        if (!$coveredHostname) continue;

        // Covered host defines its own recovery profile — read from their conf
        $coveredRaw = vv_read_conf_raw(strtolower($covered) . '.conf');
        $names = [];
        for ($tier = 1; $tier <= 4; $tier++)
            $names = array_merge($names, vv_parse_bash_array($coveredRaw, "FALLBACK_{$covered}_TIER{$tier}"));

        $active = [];
        foreach ($names as $name) {
            if (isset($running[strtolower($name)]))
                $active[] = ['name' => $name, 'image' => $running[strtolower($name)]];
        }

        if ($active) $result[] = ['host_id' => $covered, 'hostname' => $coveredHostname, 'containers' => $active];
    }

    return $result;
}

function vv_watchdog_summary(): array {
    $parseKv = function(string $raw): array {
        $out = [];
        foreach (explode("\n", $raw) as $line) {
            $line = trim($line);
            if (str_contains($line, ':'))      { [$k, $v] = explode(':', $line, 2); $out[trim($k)] = trim($v); }
            elseif (str_contains($line, '='))  { [$k, $v] = explode('=', $line, 2); $out[trim($k)] = trim($v, '"\''); }
        }
        return $out;
    };

    $dock = $parseKv(@file_get_contents(STATE_DIR . '/container_watchdog_state.db')  ?: '');
    $rw   = $parseKv(@file_get_contents(STATE_DIR . '/resource_watchdog_state.db')   ?: '');

    $ctrStrikes = [];
    foreach ($dock as $k => $v) {
        if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0)
            $ctrStrikes[$k] = (int)$v;
    }

    // Recent restarts (24 h)
    $restartLog = DB_DIR . '/container_restart_history.db';
    $restartRaw = @file_get_contents($restartLog) ?: '';
    $cutoff     = time() - 86400;
    $restarts   = [];
    foreach (explode("\n", trim($restartRaw)) as $line) {
        if (!$line || !str_contains($line, '|')) continue;
        // The second field is a formatted local date, not an epoch — see
        // vv_wd_restart_ts() in watchdog.php for why, and for what casting it with (int)
        // silently did to this list for as long as it has existed.
        [$name, $raw] = explode('|', $line, 2);
        $ts = vv_wd_restart_ts(trim($raw));
        if ($ts >= $cutoff) $restarts[] = ['name' => trim($name), 'ts' => $ts];
    }
    usort($restarts, fn($a, $b) => $b['ts'] - $a['ts']);

    // Reboots (12 h)
    $rebootRaw = @file_get_contents(STATE_DIR . '/system_watchdog_reboots.db') ?: '';
    $rbootCutoff = time() - 43200;
    $reboots = 0;
    foreach (explode("\n", trim($rebootRaw)) as $line) {
        if ((int)trim($line) >= $rbootCutoff) $reboots++;
    }

    $rwLevel       = (int)($rw['rm_action_level']  ?? 0);
    $daemonStrikes = (int)($dock['daemon_strikes']  ?? 0);
    $oomCount      = (int)trim(@file_get_contents(STATE_DIR . '/system_watchdog_oom.db') ?: '0');

    // ── Stability watchdog strikes (STATE_DIR) ───────────
    $stabRaw = @file_get_contents(STATE_DIR . '/system_watchdog_state.db') ?: '';
    $stabStrikes = [];
    foreach (explode("\n", $stabRaw) as $line) {
        $line = trim($line);
        if (!$line || !str_contains($line, ':')) continue;
        [$k, $v] = explode(':', $line, 2);
        $count = (int)trim($v);
        if ($count > 0) $stabStrikes[trim($k)] = $count;
    }

    // ── Storage watchdog strikes (STATE_DIR) ────────────
    $storRaw = @file_get_contents(STATE_DIR . '/storage_watchdog_state.db') ?: '';
    $growthStrikes = []; $logStrikes = [];
    foreach (explode("\n", $storRaw) as $line) {
        $line = trim($line);
        if (!$line || !str_contains($line, ':')) continue;
        [$k, $v] = explode(':', $line, 2);
        $count = (int)trim($v);
        if ($count <= 0) continue;
        $key = trim($k);
        if (str_starts_with($key, 'appdata_growth_'))
            $growthStrikes[substr($key, strlen('appdata_growth_'))] = $count;
        elseif (str_starts_with($key, 'appdata_log_'))
            $logStrikes[substr($key, strlen('appdata_log_'))] = $count;
    }

    // ── Network watchdog NPM strikes (STATE_DIR) ────────
    $netRaw = @file_get_contents(STATE_DIR . '/network_watchdog_state.db') ?: '';
    $npmStrikes = 0;
    foreach (explode("\n", $netRaw) as $line) {
        $line = trim($line);
        if (str_starts_with($line, 'npm:')) $npmStrikes = (int)trim(substr($line, 4));
    }

    // ── Stability live stats ──────────────────────────────────────────────────
    $dfPct = function(string $path): int {
        $out = shell_exec("df " . escapeshellarg($path) . " --output=pcent 2>/dev/null | tail -1") ?: '';
        return (int)trim(str_replace('%', '', $out));
    };

    $memRaw   = @file_get_contents('/proc/meminfo') ?: '';
    $memAvail = 0;
    if (preg_match('/^MemAvailable:\s+(\d+)/m', $memRaw, $m)) $memAvail = (int)$m[1];

    $loadRaw = @file_get_contents('/proc/loadavg') ?: '0';
    $load1   = (float)explode(' ', trim($loadRaw))[0];

    $cpuTemp = vv_cpu_temp();

    $zombies = (int)trim(shell_exec("ps -eo stat 2>/dev/null | grep -c '^Z'") ?: '0');

    $fileNr  = explode("\t", trim(@file_get_contents('/proc/sys/fs/file-nr') ?: '0	0	1'));
    $fdOpen  = max(0, (int)($fileNr[0] ?? 0) - (int)($fileNr[1] ?? 0));
    $fdMax   = max(1, (int)($fileNr[2] ?? 1));
    $fdPct   = round($fdOpen / $fdMax * 100, 1);

    $nic      = trim(shell_exec("ip route show default 2>/dev/null | awk 'NR==1{print \$5}'") ?: 'eth0') ?: 'eth0';
    $nicState = trim(@file_get_contents("/sys/class/net/$nic/operstate") ?: 'unknown');
    $sshdOk   = (int)trim(shell_exec('pgrep -c sshd 2>/dev/null') ?: '0') > 0;

    $healthy = empty($ctrStrikes) && empty($stabStrikes) && empty($growthStrikes) && empty($logStrikes)
        && $rwLevel === 0 && $daemonStrikes === 0 && $oomCount === 0 && $reboots === 0
        && $npmStrikes === 0 && $nicState === 'up' && $sshdOk;

    return [
        'healthy'        => $healthy,
        'ctr_strikes'    => $ctrStrikes,
        'rw_level'       => $rwLevel,
        'rw_paused'      => array_values(array_filter(explode(',', $rw['rm_paused_containers'] ?? ''))),
        'rw_stopped'     => array_values(array_filter(explode(',', $rw['rm_stopped_containers'] ?? ''))),
        'daemon_strikes' => $daemonStrikes,
        'oom_count'      => $oomCount,
        'reboots_12h'    => $reboots,
        'restarts_24h'   => array_slice($restarts, 0, 6),
        'restart_count'  => count($restarts),
        'stability'      => [
            'rootfs_pct'  => $dfPct('/'),
            'log_pct'     => $dfPct('/var/log'),
            'tmp_pct'     => $dfPct('/tmp'),
            'ram_free_gb' => round($memAvail / 1048576, 1),
            'load_1min'   => $load1,
            'cpu_temp'    => $cpuTemp,
            'zombies'     => $zombies,
            'fd_open'     => $fdOpen,
            'fd_max'      => $fdMax,
            'fd_pct'      => $fdPct,
            'nic'         => $nic,
            'nic_state'   => $nicState,
            'sshd_ok'     => $sshdOk,
            'strikes'     => $stabStrikes,
        ],
        'storage_wd'     => [
            'growth_strikes' => $growthStrikes,
            'log_strikes'    => $logStrikes,
        ],
        'network_wd'     => [
            'npm_strikes' => $npmStrikes,
        ],
    ];
}

// What Varaverk itself is occupying and doing, as opposed to what the machine is.
//
// The System card described the host — model, uptime, load — and said nothing about the thing
// whose dashboard it is. These are the figures that are Varaverk's own and that nothing else on
// the page reports.
//
// The cache size is the one worth watching. VV_CACHE_ROOT is /tmp/varaverk, and on Unraid /tmp is
// on rootfs, which is RAM — so this directory is memory, not disk, and the arr payload cache is
// most of it. master.conf says never to move these onto flash, which makes the size the thing to
// keep an eye on instead. Reported next to the rootfs percentage it consumes, because 187 MB
// means nothing without the ceiling it counts against.
//
// du rather than a recursive PHP walk: both roots are small and page-cached, measured at 3ms
// each, and this is assembled once a minute by the cache writer rather than per page load.
function vv_varaverk_state(): array {
    $du = function (string $path): ?int {
        if (!is_dir($path)) return null;
        $out = shell_exec('du -sb ' . escapeshellarg($path) . ' 2>/dev/null');
        return preg_match('/^(\d+)/', (string)$out, $m) ? (int)$m[1] : null;
    };

    // Locks whose process is still alive. A lock file alone does not mean a job is running —
    // acquire_lock() clears one whose pid is gone, and three were sitting in LOCK_DIR from jobs
    // that finished days ago. Counting files would have reported four jobs running and one
    // actually was.
    $running = [];
    $stale   = 0;
    foreach ((array)@glob('/tmp/unraid_locks/*.lock') as $lock) {
        $content = trim((string)@file_get_contents($lock));
        if ($content === '') { $stale++; continue; }
        [$pid, $name] = array_pad(explode(':', $content, 2), 2, '');
        $pid = (int)$pid;
        if ($pid > 1 && @posix_kill($pid, 0)) {
            $running[] = $name !== '' ? $name : basename($lock, '.lock');
        } else {
            $stale++;
        }
    }
    sort($running);

    // Newest partner conf in the RAM cache. conf_sync fills it; an age climbing past its schedule
    // means the mesh has stopped talking, which nothing else on this page would show.
    $confAge = null;
    foreach ((array)@glob(VV_CONF_RAM_CACHE_DIR . '/host*.conf') as $c) {
        $m = @filemtime($c);
        if ($m && ($confAge === null || (time() - $m) < $confAge)) $confAge = time() - $m;
    }

    // Storage mode as a fact about where this install actually is, not as the conf toggle's word
    // for it — the toggle is what someone intended and the path is what happened.
    //
    // "internal", not "flash". /boot is the internal mode in Varaverk's own vocabulary and on this
    // hardware it is a mirrored NVMe pool, not a USB stick; calling it flash on the dashboard
    // would invite a write-wear worry that does not apply here.
    $internal = str_starts_with(SCRIPTS_DIR, '/boot');

    return [
        'cache_root'   => VV_CACHE_ROOT,
        'cache_bytes'  => $du(VV_CACHE_ROOT),
        'data_bytes'   => $du(DATA_DIR),
        'scripts_dir'  => SCRIPTS_DIR,
        'storage'      => $internal ? 'internal' : 'appdata',
        'commit'       => trim((string)shell_exec(
                            'git -C ' . escapeshellarg(SCRIPTS_DIR) . ' rev-parse --short HEAD 2>/dev/null')),
        'jobs_running' => $running,
        'locks_stale'  => $stale,
        'conf_age_sec' => $confAge,
    ];
}

function vv_scripts_status(): array {
    $logDir    = LOG_DIR;
    $statFiles = array_merge(
        glob("$logDir/*.json")   ?: [],
        glob("$logDir/*/*.json") ?: []
    );

    $scripts = [];
    foreach ($statFiles as $statFile) {
        $stat   = json_decode(@file_get_contents($statFile) ?: '{}', true) ?: [];
        $status = $stat['status'] ?? 'unknown';

        // Stale running — PID gone (crash or reboot with no cleanup)
        if ($status === 'running' && !empty($stat['pid'])) {
            if (!file_exists("/proc/{$stat['pid']}")) $status = 'error';
        }

        $id   = $stat['id'] ?? basename($statFile, '.json');
        $name = basename(preg_replace('/\.sh$/', '', $id));
        $ts   = (int)($stat['end'] ?? $stat['start'] ?? @filemtime($statFile) ?: 0);

        $scripts[] = [
            'id'       => $id,
            'name'     => $name,
            'last_ts'  => $ts,
            'status'   => $status,
            'running'  => $status === 'running',
            'exit'     => $stat['exit'] ?? null,
            'duration' => isset($stat['start'], $stat['end'])
                ? (int)$stat['end'] - (int)$stat['start'] : null,
        ];
    }

    usort($scripts, fn($a, $b) => ($b['last_ts'] ?? 0) <=> ($a['last_ts'] ?? 0));
    $scripts = array_slice($scripts, 0, 12);

    return [
        'scripts'       => $scripts,
        'running_count' => count(array_filter($scripts, fn($s) => $s['status'] === 'running')),
        'ok_count'      => count(array_filter($scripts, fn($s) => $s['status'] === 'ok')),
        'warn_count'    => count(array_filter($scripts, fn($s) => $s['status'] === 'warn')),
        'error_count'   => count(array_filter($scripts, fn($s) => $s['status'] === 'error')),
    ];
}

function vv_rsync_status(): array {
    $vars = vv_conf_vars();

    $enabled = ($vars['RSYNC_ENABLED'] ?? 'true') !== 'false';

    $windows = [
        'critical'     => ($vars['CRITICAL_RSYNC_ENABLED']     ?? 'false') !== 'false',
        'daily'        => ($vars['DAILY_RSYNC_ENABLED']        ?? 'false') !== 'false',
        'intermediate' => ($vars['INTERMEDIATE_RSYNC_ENABLED'] ?? 'true')  !== 'false',
        'weekly'       => ($vars['WEEKLY_RSYNC_ENABLED']       ?? 'false') !== 'false',
        'monthly'      => ($vars['MONTHLY_RSYNC_ENABLED']      ?? 'true')  !== 'false',
    ];

    // Active rsync profiles — from lock files
    $lockDir = '/tmp/unraid_locks';
    $active  = [];
    foreach (glob("$lockDir/rsync_*.lock") ?: [] as $lf) {
        $content = trim(@file_get_contents($lf) ?: '');
        [$pid, $locked_name] = array_pad(explode(':', $content, 2), 2, '');
        if (!$pid || !file_exists("/proc/$pid")) continue;
        $profile  = preg_replace('/^rsync_/', '', $locked_name ?: basename($lf, '.lock'));
        $active[] = [
            'profile' => $profile,
            'pid'     => (int)$pid,
            'elapsed' => time() - (int)filemtime($lf),
        ];
    }

    // Last completed run per window (orchestrator log files)
    $scriptMap = [
        'critical'     => 'critical_sync_maintenance',
        'daily'        => 'daily_sync_maintenance',
        'intermediate' => 'intermediate_sync_maintenance',
        'weekly'       => 'weekly_sync_maintenance',
        'monthly'      => 'monthly_maintenance',
    ];
    $lastSync = [];
    foreach ($scriptMap as $key => $scriptName) {
        $logFile = LOG_DIR . "/Orchestrators/$scriptName.json";
        if (!file_exists($logFile)) continue;
        $stat = json_decode(@file_get_contents($logFile) ?: '{}', true) ?: [];
        $lastSync[$key] = [
            'ts'       => (int)($stat['end'] ?? $stat['start'] ?? 0),
            'status'   => $stat['status'] ?? 'unknown',
            'duration' => isset($stat['start'], $stat['end'])
                ? (int)$stat['end'] - (int)$stat['start'] : null,
        ];
    }

    // Profile activity — last 7 days, aggregated per profile
    $bwLog   = DB_DIR . '/bandwidth_history.db';
    $cutoff7 = date('Y-m-d', strtotime('-7 days'));
    $profiles = [];
    if (file_exists($bwLog)) {
        foreach (file($bwLog, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ?: [] as $line) {
            $p = explode('|', $line);
            if (count($p) < 4 || ($p[0] ?? '') < $cutoff7) continue;
            $name = $p[2] ?? '';
            if (!$name || str_ends_with($name, '-fallback')) continue;
            if (!isset($profiles[$name])) $profiles[$name] = ['runs' => 0, 'dur' => 0, 'bytes' => 0];
            $profiles[$name]['runs']++;
            $profiles[$name]['dur']   += (int)($p[3] ?? 0);
            $profiles[$name]['bytes'] += (int)($p[5] ?? 0);
        }
    }
    arsort($profiles); // sort by run count descending
    $bwSummary = array_slice($profiles, 0, 6, true);

    return [
        'enabled'    => $enabled,
        'windows'    => $windows,
        'active'     => $active,
        'last_sync'  => $lastSync,
        'bw_summary' => $bwSummary,
    ];
}