<?php
// ═══════════════════════════════════════════════════════════════════════════════════════════════
// PURPOSE
//   Probes every hostname NPM serves, from outside the proxy, and records whether it answered.
//   Produces the uptime percentages and the recent-history strip on the Proxies tab, and the list
//   of anything that had a wobble for the Sunday report.
//
// WHY IT EXISTS
//   NPM's access log says what happened when somebody visited. It says nothing at all about a host
//   nobody visited, which is exactly the host most likely to be quietly broken. Five hosts here
//   have been returning errors on every request for months without anything noticing.
//
// OPERATIONAL MODEL
//   One pass = one sample per domain, all issued in parallel through curl_multi so the wall time
//   is roughly the slowest domain rather than the sum of all of them. Intended to run every
//   minute; it is safe at any interval because everything is stored as counts and buckets rather
//   than assuming a cadence.
//
//   up   = the host answered with an HTTP status below 500.
//   down = nothing answered, the TLS handshake failed, or it answered 5xx.
//
//   A 401 or a 302 to the Authelia portal is UP. The question this asks is "is the server there",
//   and an auth redirect is the strongest possible evidence that it is. Counting a protected site
//   as down would mark every guarded host on this installation permanently offline.
//
// DESIGN PRINCIPLES
//   Probes carry a User-Agent that the access-log aggregator ignores.
//       Thirty-five domains once a minute is fifty thousand requests a day landing in the very
//       logs Tools/npm_access_stats.sh counts. Left unmarked, this monitor would become the
//       majority of the traffic it reports and bury whatever real use these hosts get.
//
//   Bounded storage, decided up front.
//       Per domain: lifetime counters, the last 60 samples for the strip, 48 hourly buckets,
//       30 daily ones and 12 monthly ones. That is a fixed size — a per-minute probe kept as raw
//       samples would be fifty thousand rows a day and the file would be the problem instead of
//       the answer. Each tier is what one view on the Proxies tab draws: 24h, 7d, 30d, 12 months.
//
//       Bounded in domains too: a record nothing has probed for VV_DOMAIN_DROP_DAYS is retired,
//       so hosts removed or renamed in NPM do not accumulate for ever under the live ones.
//
//   The domain list follows NPM, not a hand-kept list in conf.
//       A host added in the Proxies tab starts being probed without anyone remembering to add it
//       somewhere else. The list is cached so this does not call the NPM API every minute.
//
// OPERATIONAL SAFEGUARDS
//   Non-fatal, always: no NPM, no credentials, no domains — exits 0.
//   One pass at a time, flock non-blocking, so a slow pass cannot overlap the next minute's.
//   Store written tmp + rename, verified before it replaces the previous file.
//   HEAD, not GET, and nothing is followed — this is a liveness check, not a crawler.
//
// RUNTIME MODES
//   uptime_probe.php            one pass
//   uptime_probe.php --dry-run  probe and report, write nothing
//   uptime_probe.php --status   per-domain uptime table
//   uptime_probe.php --events   recent state changes, newest first
//
// CONFIGURATION
//   UPTIME_PROBE_ENABLED    master switch (default true)
//   UPTIME_PROBE_TIMEOUT    seconds per domain (default 8)
//   UPTIME_PROBE_LIST_TTL   seconds to reuse the cached domain list (default 300)
//   VV_UPTIME_UA            the User-Agent, matched by npm_access_stats.php
// ═══════════════════════════════════════════════════════════════════════════════════════════════

require_once dirname(__DIR__) . '/include/auth.php';

// Shared with npm_access_stats.php, which drops any log line containing it. Changing this in one
// place and not the other turns the monitor's own traffic back into counted requests.
const VV_UPTIME_UA   = 'Varaverk-Uptime/1.0';
const VV_SAMPLES_KEEP = 60;   // one hour at a one-minute cadence
const VV_HOURS_KEEP   = 48;
const VV_DAYS_KEEP    = 30;
// A year as twelve monthly buckets rather than 365 daily ones. The yearly view is a shape — which
// months were bad — not a date lookup, and rolling the day buckets out to 365 would grow the file
// twelvefold to answer the same question at a resolution nothing displays.
const VV_MONTHS_KEEP  = 12;
// A domain removed or renamed in NPM stops being probed, and its record would otherwise sit here
// for ever — the store only ever gained keys before this. 90 days rather than something tighter
// because a host switched off for a season is a normal thing here and its history should survive
// that; a domain nobody has served for a quarter is gone for good.
const VV_DOMAIN_DROP_DAYS = 90;
const VV_EVENTS_KEEP  = 20;

$dryRun = in_array('--dry-run', $argv, true);
$status = in_array('--status',  $argv, true);
$events = in_array('--events',  $argv, true);
$report = in_array('--report',  $argv, true);

function vv_uptime_path(): string {
    return rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/uptime.json';
}

function vv_uptime_read(): array {
    $p = vv_uptime_path();
    if (!is_file($p)) return ['domains' => []];
    $j = json_decode((string) @file_get_contents($p), true);
    if (!is_array($j) || !isset($j['domains']) || !is_array($j['domains'])) return [];
    return $j;
}

function vv_uptime_write(array $d): bool {
    $p = vv_uptime_path();
    if (!is_dir(dirname($p)) && !@mkdir(dirname($p), 0755, true)) return false;
    $d['updated'] = time();
    $json = json_encode($d, JSON_UNESCAPED_SLASHES);
    if ($json === false) return false;
    $tmp = $p . '.vv.tmp';
    if (@file_put_contents($tmp, $json) === false) return false;
    if (json_decode((string) @file_get_contents($tmp), true) === null) { @unlink($tmp); return false; }
    if (!@rename($tmp, $p)) { @unlink($tmp); return false; }
    return true;
}

// ── Report ────────────────────────────────────────────────────────────────────
// Anything that was not perfect over the last seven days, for the Sunday report. Prints nothing
// and exits 0 when every domain was clean — the orchestrator's job is to be quiet on a good week,
// and a report that always says something is a report nobody reads.
if ($report) {
    $s = vv_uptime_read();
    $doms = $s['domains'] ?? [];
    if (!$doms) { exit(0); }
    $down = $wobble = [];
    foreach ($doms as $d => $r) {
        $w = vv_uptime_window($r['days'] ?? [], 7);
        if (($r['state'] ?? '') === 'down') {
            $since = !empty($r['last_change']) ? ' since ' . date('D H:i', $r['last_change']) : '';
            $down[] = sprintf('  %-34s DOWN%s — %s', $d, $since, $r['last_detail'] ?? '');
        } elseif ($w !== null && $w < 100) {
            $wobble[] = sprintf('  %-34s %.2f%% over 7 days', $d, $w);
        }
    }
    if (!$down && !$wobble) exit(0);
    echo "Uptime — 7 day review\n";
    if ($down)   { echo "\nCurrently down:\n";        foreach ($down as $l)   echo "$l\n"; }
    if ($wobble) { echo "\nNot perfect this week:\n"; foreach ($wobble as $l) echo "$l\n"; }
    // Non-zero so the calling wrapper can notify on "there is something to say" without parsing.
    exit(1);
}

// ── Status / events ───────────────────────────────────────────────────────────
if ($status || $events) {
    $s = vv_uptime_read();
    if (!$s) { echo "uptime.json is unreadable\n"; exit(0); }
    $doms = $s['domains'] ?? [];
    if ($events) {
        $all = [];
        foreach ($doms as $d => $r) foreach ($r['events'] ?? [] as $e) $all[] = $e + ['domain' => $d];
        usort($all, fn($a, $b) => $b['ts'] <=> $a['ts']);
        if (!$all) { echo "no state changes recorded\n"; exit(0); }
        foreach (array_slice($all, 0, 40) as $e)
            printf("%s  %-34s %-5s %s\n", date('Y-m-d H:i', $e['ts']), $e['domain'],
                   strtoupper($e['to']), $e['detail'] ?? '');
        exit(0);
    }
    printf("%-34s %-6s %8s %8s %8s %8s  %s\n", 'domain', 'state', '24h', '7d', '30d', '1y', 'since');
    foreach ($doms as $d => $r) {
        $h24 = vv_uptime_window($r['hours']  ?? [], 24);
        $d7  = vv_uptime_window($r['days']   ?? [], 7);
        $d30 = vv_uptime_window($r['days']   ?? [], 30);
        $y1  = vv_uptime_window($r['months'] ?? [], 12);
        printf("%-34s %-6s %8s %8s %8s %8s  %s\n", substr($d, 0, 34), $r['state'] ?? '-',
               $h24 === null ? '-' : $h24 . '%', $d7  === null ? '-' : $d7  . '%',
               $d30 === null ? '-' : $d30 . '%', $y1  === null ? '-' : $y1  . '%',
               !empty($r['last_change']) ? date('m-d H:i', $r['last_change']) : '-');
    }
    printf("\n%d domains, last pass %s\n", count($doms),
           !empty($s['last_pass']) ? date('Y-m-d H:i:s', $s['last_pass']) : 'never');
    exit(0);
}

// Shared by --status, --report and the API, and defined once in include/auth.php so the three
// callers cannot drift apart on what "the last N" means. Buckets are keyed by time, so it is a key
// sort rather than an assumption about how many samples a period should contain — a pass that did
// not run leaves no bucket rather than a zero.
function vv_uptime_window(array $buckets, int $n): ?float {
    return vv_auth_uptime_window($buckets, $n);
}

// ── One pass ──────────────────────────────────────────────────────────────────
$lock = @fopen(sys_get_temp_dir() . '/vv_uptime_probe.lock', 'c');
if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) { echo "another pass is running\n"; exit(0); }

try {
    $v = vv_conf_vars();
    if (strtolower(trim($v['UPTIME_PROBE_ENABLED'] ?? 'true')) === 'false') {
        echo "UPTIME_PROBE_ENABLED is false\n"; exit(0);
    }
    $timeout = max(2, (int) ($v['UPTIME_PROBE_TIMEOUT'] ?? 8));
    $listTtl = max(60, (int) ($v['UPTIME_PROBE_LIST_TTL'] ?? 300));

    $store   = vv_uptime_read();
    if (!$store) { echo "uptime.json is malformed — refusing to overwrite it\n"; exit(1); }

    // The domain list, refreshed occasionally rather than every minute. NPM's API is the source of
    // truth for what is being served; asking it sixty times an hour for a list that changes weekly
    // is the sort of thing that shows up later as unexplained load.
    $now = time();
    if (empty($store['list']) || ($now - ($store['list_at'] ?? 0)) > $listTtl) {
        $p = vv_npm_list_proxies();
        if ($p['ok'] ?? false) {
            $list = [];
            foreach ($p['proxies'] as $h) {
                // A disabled host is not expected to answer, so probing it would record a
                // permanent outage for something switched off on purpose.
                if (($h['enabled'] ?? true) === false) continue;
                foreach ($h['domain_names'] ?? [] as $d) {
                    $d = strtolower(trim((string) $d));
                    // A wildcard is not a hostname you can connect to.
                    if ($d !== '' && !str_contains($d, '*')) $list[$d] = true;
                }
            }
            if ($list) { $store['list'] = array_keys($list); $store['list_at'] = $now; }
        }
    }
    $domains = $store['list'] ?? [];
    if (!$domains) { echo "no domains to probe\n"; exit(0); }

    // ── Probe, in parallel ──
    $mh = curl_multi_init();
    $handles = [];
    foreach ($domains as $d) {
        $ch = curl_init('https://' . $d . '/');
        curl_setopt_array($ch, [
            CURLOPT_NOBODY         => true,          // HEAD — liveness, not content
            CURLOPT_FOLLOWLOCATION => false,         // a redirect to the auth portal is the answer
            CURLOPT_TIMEOUT        => $timeout,
            CURLOPT_CONNECTTIMEOUT => min($timeout, 5),
            CURLOPT_USERAGENT      => VV_UPTIME_UA,
            CURLOPT_RETURNTRANSFER => true,
            // These are this host's own certificates behind its own proxy. Verification is left on
            // because a cert that stopped validating is exactly the failure worth catching.
            CURLOPT_SSL_VERIFYPEER => true,
            CURLOPT_SSL_VERIFYHOST => 2,
        ]);
        curl_multi_add_handle($mh, $ch);
        $handles[$d] = $ch;
    }
    $t0 = microtime(true);
    $running = null;
    do {
        curl_multi_exec($mh, $running);
        if ($running) curl_multi_select($mh, 1.0);
    } while ($running);

    $upN = $downN = 0;
    $downList = [];
    $changes = [];
    $doms = $store['domains'] ?? [];
    $hourKey = date('YmdH', $now);
    $dayKey  = date('Ymd',  $now);
    $monKey  = date('Ym',   $now);

    foreach ($handles as $d => $ch) {
        $errno = curl_errno($ch);
        $code  = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
        $ms    = (int) round(curl_getinfo($ch, CURLINFO_TOTAL_TIME) * 1000);
        $err   = curl_error($ch);
        curl_multi_remove_handle($mh, $ch);
        curl_close($ch);

        $up     = ($errno === 0 && $code > 0 && $code < 500);
        $detail = $errno !== 0 ? ($err ?: 'connection failed') : ('HTTP ' . $code);
        $up ? $upN++ : $downN++;
        if (!$up) $downList[] = "$d — $detail";

        $r = $doms[$d] ?? ['checks' => 0, 'up' => 0, 'down' => 0, 'state' => null,
                           'last_change' => null, 'samples' => [], 'hours' => [], 'days' => [],
                           'months' => [], 'events' => [], 'since' => $now];
        $r['checks']++;
        $up ? $r['up']++ : $r['down']++;
        $r['last_code'] = $code;
        $r['last_ms']   = $ms;
        $r['last_at']   = $now;
        $r['last_detail'] = $detail;

        $newState = $up ? 'up' : 'down';
        if (($r['state'] ?? null) !== $newState) {
            // The first observation is not a transition — there was no previous state to leave.
            if ($r['state'] !== null) {
                $ev = ['ts' => $now, 'to' => $newState, 'detail' => $detail];
                $r['events'][] = $ev;
                $r['events'] = array_slice($r['events'], -VV_EVENTS_KEEP);
                $changes[] = "$d → " . strtoupper($newState) . " ($detail)";
            }
            $r['state'] = $newState;
            $r['last_change'] = $now;
        }

        $r['samples'][] = $up ? 1 : 0;
        if (count($r['samples']) > VV_SAMPLES_KEEP)
            $r['samples'] = array_slice($r['samples'], -VV_SAMPLES_KEEP);

        foreach ([['hours',  $hourKey, VV_HOURS_KEEP],
                  ['days',   $dayKey,  VV_DAYS_KEEP],
                  ['months', $monKey,  VV_MONTHS_KEEP]] as [$k, $key, $keep]) {
            $b = $r[$k][$key] ?? ['u' => 0, 't' => 0];
            $b['t']++;
            if ($up) $b['u']++;
            $r[$k][$key] = $b;
            if (count($r[$k]) > $keep) { krsort($r[$k]); $r[$k] = array_slice($r[$k], 0, $keep, true); ksort($r[$k]); }
        }

        $doms[$d] = $r;
    }
    curl_multi_close($mh);

    // Retire domains nothing has probed in a long time. Keyed on last_at rather than on absence
    // from the current list, which matters when NPM is unreachable: the cached list keeps being
    // probed, every domain keeps getting a last_at, and an NPM outage therefore cannot empty the
    // store. Only a domain that genuinely left the list stops being stamped.
    $cutoff = $now - (VV_DOMAIN_DROP_DAYS * 86400);
    $dropped = [];
    foreach ($doms as $d => $r) {
        // A record with no last_at at all is kept. It should not be possible — every probe stamps
        // it — and deleting on missing data is the wrong way round for something irreversible.
        if (isset($r['last_at']) && $r['last_at'] < $cutoff) { unset($doms[$d]); $dropped[] = $d; }
    }

    $store['domains']   = $doms;
    $store['last_pass'] = $now;

    printf("%d domains — %d up, %d down, %.1fs%s\n", count($handles), $upN, $downN,
           microtime(true) - $t0, $changes ? '' : ' (no state changes)');
    foreach ($changes as $c) echo "  $c\n";
    foreach ($downList as $c) echo "  DOWN $c\n";
    // Named, not silent. Dropping a record throws away months of history, and a line in the log is
    // the only trace that it was this and not the store being reset by something else.
    foreach ($dropped as $d)
        echo "  RETIRED $d — not probed in " . VV_DOMAIN_DROP_DAYS . " days\n";

    if ($dryRun) { echo "dry run — nothing written\n"; exit(0); }
    if (!vv_uptime_write($store)) { echo 'could not write ' . vv_uptime_path() . "\n"; exit(1); }
    exit(0);

} finally {
    flock($lock, LOCK_UN);
    fclose($lock);
}