diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 7debb44..f7058d4 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -587,6 +587,7 @@ "Monitors/zfs_memory_snapshot.sh" # ZFS pool health + ARC + Docker memory snapshot "Monitors/smart_health.sh" # drive SMART attributes — reallocated, pending, temp "Monitors/cert_monitor.sh" # SSL certificate expiry for all configured domains + "Monitors/uptime_report.sh" # domains down now, and any that were not 100% this week "Monitors/backup_verify.sh" # rsync mirror integrity via independent MD5 checksums "Monitors/bandwidth_monitor.sh" # weekly rsync transfer totals and per-share breakdown "Monitors/emby_session_report.sh" # Emby usage — streams, users, library, transcode ratio @@ -1482,6 +1483,21 @@ # not deleted: the history of something that broke is the reason the file is kept. CERT_HISTORY_STRIKES=5 # expired passes before a domain is retired from the active list +# ── Uptime Probe ── +# Tools/uptime_probe.sh checks every hostname NPM serves, once a minute, from outside the proxy. +# NPM's access log only describes hosts somebody visited; this is what watches the ones nobody +# does — which is where a quietly broken container hides. +# +# A 302 to the Authelia portal or a 401 counts as UP. The question is whether the server is there, +# and an auth redirect is proof that it is; counting it as down would mark every protected host on +# this mesh permanently offline. +# +# Probes carry the User-Agent Varaverk-Uptime/1.0, which npm_access_stats.sh excludes from its +# request counts. Fifty thousand self-inflicted requests a day would otherwise bury real traffic. + UPTIME_PROBE_ENABLED=true # master switch for the per-minute probe + UPTIME_PROBE_TIMEOUT=8 # seconds per domain before it counts as down + UPTIME_PROBE_LIST_TTL=900 # seconds to reuse the domain list from NPM before re-reading it + # ━━━ Backup Verify ━━━ # Verifies rsync mirror health by comparing random file checksums between servers. # Catches silent corruption or incomplete syncs that rsync itself wouldn't detect. diff --git a/Monitors/uptime_report.sh b/Monitors/uptime_report.sh new file mode 100755 index 0000000..32eeb44 --- /dev/null +++ b/Monitors/uptime_report.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# ============================================================================================== +# ================================== Uptime Report ============================================= +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# The weekly read of what Tools/uptime_probe.sh has been recording every minute: anything down +# right now, and anything that was not perfect over the last seven days. Runs in the Sunday +# Morning Coffee Report. +# +# Silent on a clean week. A report that always says something is a report nobody reads, so this +# prints nothing and notifies nothing when every domain was 100%. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# uptime_probe.php --report exits 1 when it has something to say and 0 when it does not, so the +# decision to notify is the exit code rather than this script parsing the text it just printed. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# UPTIME_PROBE_ENABLED nothing here runs when the probe is switched off +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../load_config.sh" + +acquire_lock +detect_hosts + +if [[ "${UPTIME_PROBE_ENABLED:-true}" == "false" ]]; then + log "$ICON_GEAR Uptime probe disabled — nothing to report" + exit 0 +fi + +REPORT="$(php "$SCRIPT_DIR/../Plugin/unraid/Tools/uptime_probe.php" --report 2>/dev/null)" +RC=$? + +if [[ $RC -eq 0 || -z "$REPORT" ]]; then + echo "$ICON_DONE All monitored domains at 100% this week ✅" + exit 0 +fi + +echo "$REPORT" + +DOWN_COUNT=$(grep -c "DOWN" <<< "$REPORT" || true) +if [[ "$DOWN_COUNT" -gt 0 ]]; then + notify "$DOWN_COUNT domain(s) currently unreachable on $(hostname)" "Uptime" "warning" +else + notify "Some domains had downtime this week on $(hostname)" "Uptime" "normal" +fi +exit 0 diff --git a/Plugin/unraid/Tools/npm_access_stats.php b/Plugin/unraid/Tools/npm_access_stats.php index 00ae377..d57da84 100644 --- a/Plugin/unraid/Tools/npm_access_stats.php +++ b/Plugin/unraid/Tools/npm_access_stats.php @@ -149,6 +149,7 @@ try { $t0 = microtime(true); $readTotal = 0; $newLines = 0; + $skipped = 0; foreach ($files as $f) { if (!preg_match('/proxy-host-(\d+)_access\.log$/', $f, $m)) continue; @@ -185,6 +186,13 @@ try { if (substr($line, -1) !== "\n") break; $lastCompleteOffset += $len; + // Varaverk's own uptime probe, dropped before it is counted. It hits every host once a + // minute — fifty thousand requests a day — so counting it would make this monitor the + // overwhelming majority of the traffic it reports, and a host nobody visits would look + // as busy as one that is genuinely used. The string is VV_UPTIME_UA in uptime_probe.php + // and the two must stay in step. + if (strpos($line, 'Varaverk-Uptime/') !== false) { $skipped++; continue; } + // [09/Aug/2026:05:52:08 +0000] - 200 200 - GET https host "/" [Client 1.2.3.4] [Length 567] ... if (!preg_match('/^\[([^\]]+)\]\s+\S+\s+(\d{3})/', $line, $lm)) continue; $code = (int) $lm[2]; @@ -211,8 +219,9 @@ try { $store['hosts'] = $hosts; $store['last_pass'] = time(); - printf("%d logs, read %s this pass, %s new requests, %.1fs\n", - count($files), vv_npm_bytes($readTotal), number_format($newLines), microtime(true) - $t0); + printf("%d logs, read %s this pass, %s new requests, %s own probes ignored, %.1fs\n", + count($files), vv_npm_bytes($readTotal), number_format($newLines), + number_format($skipped), microtime(true) - $t0); if ($dryRun) { echo "dry run — nothing written\n"; exit(0); } if (!vv_npm_stats_write($store)) { echo 'could not write ' . vv_npm_stats_path() . "\n"; exit(1); } diff --git a/Plugin/unraid/Tools/uptime_probe.php b/Plugin/unraid/Tools/uptime_probe.php new file mode 100644 index 0000000..eeb9e92 --- /dev/null +++ b/Plugin/unraid/Tools/uptime_probe.php @@ -0,0 +1,310 @@ + []]; + $j = json_decode((string) @file_get_contents($p), true); + if (!is_array($j) || !isset($j['domains']) || !is_array($j['domains'])) return []; + return $j; +} + +function vv_uptime_write(array $d): bool { + $p = vv_uptime_path(); + if (!is_dir(dirname($p)) && !@mkdir(dirname($p), 0755, true)) return false; + $d['updated'] = time(); + $json = json_encode($d, JSON_UNESCAPED_SLASHES); + if ($json === false) return false; + $tmp = $p . '.vv.tmp'; + if (@file_put_contents($tmp, $json) === false) return false; + if (json_decode((string) @file_get_contents($tmp), true) === null) { @unlink($tmp); return false; } + if (!@rename($tmp, $p)) { @unlink($tmp); return false; } + return true; +} + +function vv_uptime_pct(int $up, int $total): ?float { + return $total > 0 ? round($up / $total * 100, 2) : null; +} + +// ── Report ──────────────────────────────────────────────────────────────────── +// Anything that was not perfect over the last seven days, for the Sunday report. Prints nothing +// and exits 0 when every domain was clean — the orchestrator's job is to be quiet on a good week, +// and a report that always says something is a report nobody reads. +if ($report) { + $s = vv_uptime_read(); + $doms = $s['domains'] ?? []; + if (!$doms) { exit(0); } + $down = $wobble = []; + foreach ($doms as $d => $r) { + $w = vv_uptime_window($r['days'] ?? [], 7); + if (($r['state'] ?? '') === 'down') { + $since = !empty($r['last_change']) ? ' since ' . date('D H:i', $r['last_change']) : ''; + $down[] = sprintf(' %-34s DOWN%s — %s', $d, $since, $r['last_detail'] ?? ''); + } elseif ($w !== null && $w < 100) { + $wobble[] = sprintf(' %-34s %.2f%% over 7 days', $d, $w); + } + } + if (!$down && !$wobble) exit(0); + echo "Uptime — 7 day review\n"; + if ($down) { echo "\nCurrently down:\n"; foreach ($down as $l) echo "$l\n"; } + if ($wobble) { echo "\nNot perfect this week:\n"; foreach ($wobble as $l) echo "$l\n"; } + // Non-zero so the calling wrapper can notify on "there is something to say" without parsing. + exit(1); +} + +// ── Status / events ─────────────────────────────────────────────────────────── +if ($status || $events) { + $s = vv_uptime_read(); + if (!$s) { echo "uptime.json is unreadable\n"; exit(0); } + $doms = $s['domains'] ?? []; + if ($events) { + $all = []; + foreach ($doms as $d => $r) foreach ($r['events'] ?? [] as $e) $all[] = $e + ['domain' => $d]; + usort($all, fn($a, $b) => $b['ts'] <=> $a['ts']); + if (!$all) { echo "no state changes recorded\n"; exit(0); } + foreach (array_slice($all, 0, 40) as $e) + printf("%s %-34s %-5s %s\n", date('Y-m-d H:i', $e['ts']), $e['domain'], + strtoupper($e['to']), $e['detail'] ?? ''); + exit(0); + } + printf("%-34s %-6s %8s %8s %8s %s\n", 'domain', 'state', '1h', '24h', '30d', 'since'); + foreach ($doms as $d => $r) { + $h = vv_uptime_window($r['hours'] ?? [], 1); + $h24 = vv_uptime_window($r['hours'] ?? [], 24); + $d30 = vv_uptime_window($r['days'] ?? [], 30); + printf("%-34s %-6s %8s %8s %8s %s\n", substr($d, 0, 34), $r['state'] ?? '-', + $h === null ? '-' : $h . '%', $h24 === null ? '-' : $h24 . '%', + $d30 === null ? '-' : $d30 . '%', + !empty($r['last_change']) ? date('m-d H:i', $r['last_change']) : '-'); + } + printf("\n%d domains, last pass %s\n", count($doms), + !empty($s['last_pass']) ? date('Y-m-d H:i:s', $s['last_pass']) : 'never'); + exit(0); +} + +// Shared by --status, --report and the API. Buckets are keyed by time so "the last N" is a key sort, not an assumption about how many +// samples a period should contain — a pass that did not run leaves no bucket rather than a zero. +function vv_uptime_window(array $buckets, int $n): ?float { + if (!$buckets) return null; + krsort($buckets); + $u = $t = 0; + foreach (array_slice($buckets, 0, $n, true) as $b) { $u += $b['u'] ?? 0; $t += $b['t'] ?? 0; } + return vv_uptime_pct($u, $t); +} + +// ── One pass ────────────────────────────────────────────────────────────────── +$lock = @fopen(sys_get_temp_dir() . '/vv_uptime_probe.lock', 'c'); +if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) { echo "another pass is running\n"; exit(0); } + +try { + $v = vv_conf_vars(); + if (strtolower(trim($v['UPTIME_PROBE_ENABLED'] ?? 'true')) === 'false') { + echo "UPTIME_PROBE_ENABLED is false\n"; exit(0); + } + $timeout = max(2, (int) ($v['UPTIME_PROBE_TIMEOUT'] ?? 8)); + $listTtl = max(60, (int) ($v['UPTIME_PROBE_LIST_TTL'] ?? 900)); + + $store = vv_uptime_read(); + if (!$store) { echo "uptime.json is malformed — refusing to overwrite it\n"; exit(1); } + + // The domain list, refreshed occasionally rather than every minute. NPM's API is the source of + // truth for what is being served; asking it sixty times an hour for a list that changes weekly + // is the sort of thing that shows up later as unexplained load. + $now = time(); + if (empty($store['list']) || ($now - ($store['list_at'] ?? 0)) > $listTtl) { + $p = vv_npm_list_proxies(); + if ($p['ok'] ?? false) { + $list = []; + foreach ($p['proxies'] as $h) { + // A disabled host is not expected to answer, so probing it would record a + // permanent outage for something switched off on purpose. + if (($h['enabled'] ?? true) === false) continue; + foreach ($h['domain_names'] ?? [] as $d) { + $d = strtolower(trim((string) $d)); + // A wildcard is not a hostname you can connect to. + if ($d !== '' && !str_contains($d, '*')) $list[$d] = true; + } + } + if ($list) { $store['list'] = array_keys($list); $store['list_at'] = $now; } + } + } + $domains = $store['list'] ?? []; + if (!$domains) { echo "no domains to probe\n"; exit(0); } + + // ── Probe, in parallel ── + $mh = curl_multi_init(); + $handles = []; + foreach ($domains as $d) { + $ch = curl_init('https://' . $d . '/'); + curl_setopt_array($ch, [ + CURLOPT_NOBODY => true, // HEAD — liveness, not content + CURLOPT_FOLLOWLOCATION => false, // a redirect to the auth portal is the answer + CURLOPT_TIMEOUT => $timeout, + CURLOPT_CONNECTTIMEOUT => min($timeout, 5), + CURLOPT_USERAGENT => VV_UPTIME_UA, + CURLOPT_RETURNTRANSFER => true, + // These are this host's own certificates behind its own proxy. Verification is left on + // because a cert that stopped validating is exactly the failure worth catching. + CURLOPT_SSL_VERIFYPEER => true, + CURLOPT_SSL_VERIFYHOST => 2, + ]); + curl_multi_add_handle($mh, $ch); + $handles[$d] = $ch; + } + $t0 = microtime(true); + $running = null; + do { + curl_multi_exec($mh, $running); + if ($running) curl_multi_select($mh, 1.0); + } while ($running); + + $upN = $downN = 0; + $downList = []; + $changes = []; + $doms = $store['domains'] ?? []; + $hourKey = date('YmdH', $now); + $dayKey = date('Ymd', $now); + + foreach ($handles as $d => $ch) { + $errno = curl_errno($ch); + $code = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE); + $ms = (int) round(curl_getinfo($ch, CURLINFO_TOTAL_TIME) * 1000); + $err = curl_error($ch); + curl_multi_remove_handle($mh, $ch); + curl_close($ch); + + $up = ($errno === 0 && $code > 0 && $code < 500); + $detail = $errno !== 0 ? ($err ?: 'connection failed') : ('HTTP ' . $code); + $up ? $upN++ : $downN++; + if (!$up) $downList[] = "$d — $detail"; + + $r = $doms[$d] ?? ['checks' => 0, 'up' => 0, 'down' => 0, 'state' => null, + 'last_change' => null, 'samples' => [], 'hours' => [], 'days' => [], + 'events' => [], 'since' => $now]; + $r['checks']++; + $up ? $r['up']++ : $r['down']++; + $r['last_code'] = $code; + $r['last_ms'] = $ms; + $r['last_at'] = $now; + $r['last_detail'] = $detail; + + $newState = $up ? 'up' : 'down'; + if (($r['state'] ?? null) !== $newState) { + // The first observation is not a transition — there was no previous state to leave. + if ($r['state'] !== null) { + $ev = ['ts' => $now, 'to' => $newState, 'detail' => $detail]; + $r['events'][] = $ev; + $r['events'] = array_slice($r['events'], -VV_EVENTS_KEEP); + $changes[] = "$d → " . strtoupper($newState) . " ($detail)"; + } + $r['state'] = $newState; + $r['last_change'] = $now; + } + + $r['samples'][] = $up ? 1 : 0; + if (count($r['samples']) > VV_SAMPLES_KEEP) + $r['samples'] = array_slice($r['samples'], -VV_SAMPLES_KEEP); + + foreach ([['hours', $hourKey, VV_HOURS_KEEP], ['days', $dayKey, VV_DAYS_KEEP]] as [$k, $key, $keep]) { + $b = $r[$k][$key] ?? ['u' => 0, 't' => 0]; + $b['t']++; + if ($up) $b['u']++; + $r[$k][$key] = $b; + if (count($r[$k]) > $keep) { krsort($r[$k]); $r[$k] = array_slice($r[$k], 0, $keep, true); ksort($r[$k]); } + } + + $doms[$d] = $r; + } + curl_multi_close($mh); + + $store['domains'] = $doms; + $store['last_pass'] = $now; + + printf("%d domains — %d up, %d down, %.1fs%s\n", count($handles), $upN, $downN, + microtime(true) - $t0, $changes ? '' : ' (no state changes)'); + foreach ($changes as $c) echo " $c\n"; + foreach ($downList as $c) echo " DOWN $c\n"; + + if ($dryRun) { echo "dry run — nothing written\n"; exit(0); } + if (!vv_uptime_write($store)) { echo 'could not write ' . vv_uptime_path() . "\n"; exit(1); } + exit(0); + +} finally { + flock($lock, LOCK_UN); + fclose($lock); +} diff --git a/Plugin/unraid/Tools/uptime_probe.sh b/Plugin/unraid/Tools/uptime_probe.sh new file mode 100755 index 0000000..ea17b53 --- /dev/null +++ b/Plugin/unraid/Tools/uptime_probe.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# ============================================================================================== +# ==================================== Uptime Probe ============================================ +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Probes every hostname NPM serves, from outside the proxy, once per run. Produces the uptime +# percentages and history strip on the Proxies tab, and the wobble list for the Sunday report. +# +# NPM's access log only describes hosts somebody visited. The host most likely to be quietly +# broken is the one nobody visited, and nothing was watching those at all. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# A wrapper. The work is in uptime_probe.php, which issues all probes in parallel through +# curl_multi so one pass costs about as long as the slowest domain rather than the sum. +# +# Runs every minute, injected by include/scheduler.php alongside the other background writers. +# Nothing about the storage assumes that cadence — everything is counts and time buckets. +# +# Probes carry the User-Agent Varaverk-Uptime/1.0, which npm_access_stats.sh excludes. Without +# that, this monitor becomes fifty thousand requests a day in the logs it reports on. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# uptime_probe.sh one pass +# uptime_probe.sh --dry-run probe and report, write nothing +# uptime_probe.sh --status per-domain uptime table (1h / 24h / 30d) +# uptime_probe.sh --events recent state changes, newest first +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# UPTIME_PROBE_ENABLED master switch +# UPTIME_PROBE_TIMEOUT seconds per domain +# UPTIME_PROBE_LIST_TTL seconds to reuse the cached domain list from NPM +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +php "$SCRIPT_DIR/uptime_probe.php" "$@" diff --git a/Plugin/unraid/api/auth.php b/Plugin/unraid/api/auth.php index 3e585b9..b050c26 100644 --- a/Plugin/unraid/api/auth.php +++ b/Plugin/unraid/api/auth.php @@ -102,7 +102,7 @@ require_once dirname(__DIR__) . '/include/auth.php'; // panels every stack carries rather than gated to one. const VV_AUTH_ACTION_PANEL = [ // GET - 'npm_proxies' => 'proxies', 'npm_certs' => 'proxies', 'npm_stats' => 'proxies', + 'npm_proxies' => 'proxies', 'npm_certs' => 'proxies', 'npm_stats' => 'proxies', 'npm_uptime' => 'proxies', 'lldap_users' => 'users', 'lldap_groups' => 'users', 'lldap_avatar' => 'users', 'authelia_rules' => 'acl', // POST @@ -115,6 +115,16 @@ const VV_AUTH_ACTION_PANEL = [ 'authelia_save' => 'acl', ]; +// Same windowing rule as uptime_probe.php: buckets are keyed by time, so "the last N" is a key +// sort rather than an assumption that every period produced a sample. +function vv_uptime_window_api(array $buckets, int $n): ?float { + if (!$buckets) return null; + krsort($buckets); + $u = $t = 0; + foreach (array_slice($buckets, 0, $n, true) as $b) { $u += $b['u'] ?? 0; $t += $b['t'] ?? 0; } + return $t > 0 ? round($u / $t * 100, 2) : null; +} + function vv_auth_action_allowed(string $action): bool { $panel = VV_AUTH_ACTION_PANEL[$action] ?? null; // Unmapped actions are left to the existing "Unknown action" answer rather than being refused @@ -161,6 +171,30 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') { exit; } + // Per-domain uptime, written by Tools/uptime_probe.sh every minute. Keyed by hostname rather + // than proxy id, because that is what was probed. + if ($action === 'npm_uptime') { + $f = rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/uptime.json'; + $u = is_file($f) ? (json_decode((string) @file_get_contents($f), true) ?: []) : []; + $out = []; + foreach ($u['domains'] ?? [] as $dom => $r) { + // Only what the row draws. The hourly and daily buckets are dozens of entries per + // domain and the page shows three percentages and a strip. + $out[$dom] = [ + 'state' => $r['state'] ?? null, + 'samples' => array_slice($r['samples'] ?? [], -60), + 'h1' => vv_uptime_window_api($r['hours'] ?? [], 1), + 'h24' => vv_uptime_window_api($r['hours'] ?? [], 24), + 'd30' => vv_uptime_window_api($r['days'] ?? [], 30), + 'last_change' => $r['last_change'] ?? null, + 'last_detail' => $r['last_detail'] ?? null, + 'last_ms' => $r['last_ms'] ?? null, + ]; + } + echo json_encode(['ok' => true, 'domains' => $out, 'last_pass' => $u['last_pass'] ?? null]); + exit; + } + $result = match ($action) { 'npm_proxies' => vv_npm_list_proxies(), 'npm_certs' => ['ok' => true, 'certs' => vv_npm_list_certs()], diff --git a/Plugin/unraid/include/scheduler.php b/Plugin/unraid/include/scheduler.php index 01ed508..945c46b 100644 --- a/Plugin/unraid/include/scheduler.php +++ b/Plugin/unraid/include/scheduler.php @@ -189,6 +189,10 @@ function vv_cron_rebuild(array $schedule): bool { foreach ([ ['* * * * *', 'api_cache_writer.sh'], ['0 */2 * * *', 'remote_arr_cache_writer.sh'], + // Every minute, and it gates itself on UPTIME_PROBE_ENABLED rather than being removed from + // here when switched off — a cron entry that appears and disappears is harder to reason + // about than one that always exists and sometimes exits immediately. + ['* * * * *', 'uptime_probe.sh'], ] as [$cron, $script]) { $path = "$toolsDir/$script"; if (file_exists($path)) $lines[] = "$cron bash \"$runner\" \"Plugin/unraid/Tools/$script\" \"$path\""; diff --git a/Plugin/unraid/pages/auth.php b/Plugin/unraid/pages/auth.php index a353d30..652088a 100644 --- a/Plugin/unraid/pages/auth.php +++ b/Plugin/unraid/pages/auth.php @@ -266,6 +266,15 @@ require_once dirname(__DIR__) . '/include/ai_chat.php'; .vv-au-stat-s { font-size:9px;color:#3a3a3a;margin-top:1px; } .vv-au-stat-s.warn { color:#8a6a2a; } .vv-au-stat-s.bad { color:#a34; } +/* Healthy uptime is deliberately quiet — a page where thirty green numbers shout is a page where + the two red ones do not. */ +.vv-au-stat-n.ok2 { color:#4a7a4a; } + +/* One bar per probe, last hour. The shape is the part a percentage throws away: one long outage + and sixty scattered blips are the same 50% and completely different problems. */ +.vv-au-spark { display:flex;gap:1px;justify-content:flex-end;align-items:flex-end;height:11px;margin-top:3px; } +.vv-au-spark i { width:2px;height:100%;background:#2d5a2d;border-radius:1px;flex-shrink:0; } +.vv-au-spark i.d{ background:#ef5350; } /* A disabled host is still listed — it is a thing you might re-enable — but it should not read as part of what is currently serving. */ .vv-au-off td { opacity:.42; } @@ -354,7 +363,7 @@ $tabs = ['proxies' => 'Proxies', 'users' => 'Users & Groups',
Loading…
- + @@ -510,6 +519,8 @@ const IS_OWNER = ; let _proxies = [], _certs = []; // Keyed by proxy-host id, from Tools/npm_access_stats.sh. Empty until it has run once. let _proxyStats = {}; +// Keyed by hostname, from Tools/uptime_probe.sh. Empty until the probe has run once. +let _uptime = {}; let _users = [], _groups = []; let _rules = [], _defaultPolicy = 'deny'; // The trailing comment on the default_policy line, carried so a save puts it back. The block is @@ -646,7 +657,8 @@ function _loadProxies() { // Stats are optional decoration — the list must render whether or not the aggregator has run, // so this neither blocks _check() nor fails the load. - _get('npm_stats', r => { _proxyStats = (r && r.ok && r.hosts) ? r.hosts : {}; if (proxiesLoaded) _renderProxies(); }); + _get('npm_stats', r => { _proxyStats = (r && r.ok && r.hosts) ? r.hosts : {}; if (proxiesLoaded) _renderProxies(); }); + _get('npm_uptime', r => { _uptime = (r && r.ok && r.domains) ? r.domains : {}; if (proxiesLoaded) _renderProxies(); }); _get('npm_certs', r => { _certs = r.certs || []; certsLoaded = true; _check(); }); _get('npm_proxies', r => { if (!r.ok) { loading.innerHTML = ''+_esc(r.error)+''; return; } @@ -716,6 +728,7 @@ function _renderProxies() {
→ ${_esc(fwd)}
${marks}
+ ${_upCell(p)} ${reqCell} ${errCell} ${_togHtml('proxy-' + p.id, enabled, enabled ? 'Enabled — click to disable' : 'Disabled — click to enable')} @@ -729,6 +742,33 @@ function _renderProxies() { }).join(''); } +// Uptime for the host, from Tools/uptime_probe.sh. A proxy host can carry several domains, so the +// worst of them is what the row reports — a host is only as reachable as its least reachable name, +// and averaging would hide one dead domain behind three healthy ones. +function _upCell(p) { + const doms = (p.domain_names || []).map(d => String(d).toLowerCase()).filter(d => !d.includes('*')); + const recs = doms.map(d => _uptime[d]).filter(Boolean); + if (!recs.length) return '
—
'; + + const worst = recs.reduce((a, b) => ((a.h24 ?? 101) <= (b.h24 ?? 101) ? a : b)); + const down = recs.some(r => r.state === 'down'); + const pct = worst.h24; + // Banded on the same three-state rule as the error column: fine, worth a look, not working. + const cls = down ? ' bad' : (pct === null ? ' dim' : (pct >= 99.5 ? ' ok2' : (pct >= 95 ? ' warn' : ' bad'))); + + // Sixty samples is the last hour at a one-minute cadence. Drawn as bars rather than a number + // because the shape — one long outage or sixty scattered blips — is the part a percentage loses. + const bars = (worst.samples || []).map(s => + ``).join(''); + + const title = down + ? 'DOWN — ' + (worst.last_detail || 'no response') + : 'up' + (worst.last_ms ? ' · ' + worst.last_ms + 'ms' : ''); + + return `
${pct === null ? '—' : (pct >= 99.95 ? '100' : pct.toFixed(pct >= 99 ? 2 : 1))}% 24h
+
${bars}
`; +} + function _certName(id) { const c = _certs.find(x => String(x.id) === String(id)); return c ? (c.nice_name || (c.domain_names || []).join(', ')) : 'certificate ' + id; diff --git a/Plugin/unraid/pages/readme/ui-map.md b/Plugin/unraid/pages/readme/ui-map.md index 633cc6d..449176b 100644 --- a/Plugin/unraid/pages/readme/ui-map.md +++ b/Plugin/unraid/pages/readme/ui-map.md @@ -1996,6 +1996,18 @@ Saved into `master.conf`, which does not need to be opened by hand. | `TRANSCODE_LOG_RETENTION` | a number box, in days | in this section | days before old entries purged | | `TRANSCODE_CHECK_EMBY` | a switch | in this section | — | +## Uptime Probe + +Route: Settings tab → All settings → *Uptime Probe* + +Saved into `master.conf`, which does not need to be opened by hand. + +| Setting | Control | Where | What it does | +|---|---|---|---| +| `UPTIME_PROBE_ENABLED` | a switch | in this section | master switch for the per-minute probe | +| `UPTIME_PROBE_TIMEOUT` | a number box, in seconds | in this section | seconds per domain before it counts as down | +| `UPTIME_PROBE_LIST_TTL` | a number box, in seconds | in this section | seconds to reuse the domain list from NPM before re-reading it | + ## Version Parity Route: Settings tab → All settings → *Version Parity*