diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 949a155..31ca137 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -488,6 +488,11 @@ "Arrs_Stack/sonarr_tvdb_removed.sh" # remove series dropped from TVDB "Docker_Essentials/docker_update.sh" # pull container image updates before restart "Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last + # Last, and after git_pull_execute.sh which must always run first. Daily rather than with + # the Sunday cert monitor: CERT_HISTORY_STRIKES counts passes, so a weekly cadence would + # make five strikes five weeks. One NPM list call; it is what gives the Certs tab its + # per-domain renewal and failure counts. + "Plugin/unraid/Tools/cert_history.sh" # record cert renewals, failures and age per domain ) # Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart. @@ -1469,6 +1474,13 @@ CERT_CRIT_DAYS=7 # critical alert within this many days CERT_TIMEOUT=10 # seconds per domain before giving up +# ── Certificate History ── +# Tools/cert_history.sh records every certificate NPM holds — first seen, renewals, failures — +# into DB_DIR/cert_history.json, which is what the Certs tab draws its per-domain cards from. +# A domain found expired on this many consecutive passes stops being counted as working. It is +# not deleted: the history of something that broke is the reason the file is kept. + CERT_HISTORY_STRIKES=5 # expired passes before a domain is retired from the active list + # ━━━ Backup Verify ━━━ # Verifies rsync mirror health by comparing random file checksums between servers. # Catches silent corruption or incomplete syncs that rsync itself wouldn't detect. diff --git a/Plugin/unraid/Tools/cert_history.php b/Plugin/unraid/Tools/cert_history.php new file mode 100644 index 0000000..3526c1c --- /dev/null +++ b/Plugin/unraid/Tools/cert_history.php @@ -0,0 +1,292 @@ + [], 'created' => time()]; + $j = json_decode((string) @file_get_contents($p), true); + // A corrupt store is not overwritten from here — it is reported and left alone, because the + // alternative is a pass that silently restarts every counter from zero. + if (!is_array($j) || !isset($j['domains']) || !is_array($j['domains'])) return []; + return $j; +} + +function vv_cert_history_write(array $data): bool { + $p = vv_cert_history_path(); + $dir = dirname($p); + if (!is_dir($dir) && !@mkdir($dir, 0755, true)) return false; + $data['updated'] = time(); + $json = json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + if ($json === false) return false; + $tmp = $p . '.vv.tmp'; + if (@file_put_contents($tmp, $json) === false) return false; + // Verified before it replaces the real file. This is the only copy of the history. + if (json_decode((string) @file_get_contents($tmp), true) === null) { @unlink($tmp); return false; } + if (!@rename($tmp, $p)) { @unlink($tmp); return false; } + return true; +} + +function vv_cert_strike_limit(): int { + $n = (int) (vv_conf_vars()['CERT_HISTORY_STRIKES'] ?? 5); + return $n > 0 ? $n : 5; +} + +// ── Status ──────────────────────────────────────────────────────────────────── +if ($status) { + $h = vv_cert_history_read(); + if (!$h) { echo "cert_history.json is unreadable or malformed\n"; exit(0); } + $d = $h['domains'] ?? []; + printf("%-32s %7s %7s %7s %7s %-10s %s\n", + 'domain', 'checks', 'renews', 'fails', 'strikes', 'expires', 'tracked'); + foreach ($d as $dom => $r) { + printf("%-32s %7d %7d %7d %7d %-10s %s%s\n", substr($dom, 0, 32), + $r['checks'] ?? 0, $r['renewals'] ?? 0, $r['failures'] ?? 0, $r['strikes'] ?? 0, + $r['last_expiry'] ?? '-', vv_cert_span($r['first_seen'] ?? time()), + !empty($r['retired_at']) ? ' RETIRED' : (!empty($r['removed_at']) ? ' removed' : '')); + } + printf("\n%d tracked, strike limit %d\n", count($d), vv_cert_strike_limit()); + exit(0); +} + +// Years, months and days rather than a day count. "3 years 6 months and 22 days" is the shape the +// question is asked in; 1298 days is the same fact in a unit nobody thinks in. +function vv_cert_span(int $from, ?int $to = null): string { + $a = (new DateTime())->setTimestamp($from); + $b = (new DateTime())->setTimestamp($to ?? time()); + if ($b < $a) return '0d'; + $d = $a->diff($b); + $out = []; + if ($d->y) $out[] = $d->y . 'y'; + if ($d->m) $out[] = $d->m . 'mo'; + if ($d->d || !$out) $out[] = $d->d . 'd'; + return implode(' ', $out); +} + +// What one observation of one domain does to its record. Pure — takes the record and the facts, +// returns the new record and what happened — so the strike ladder can be tested without waiting +// for a certificate to expire. That mattered: nothing on this host is expired right now, so the +// failure branch would otherwise ship having never run. +// +// $exp the certificate's expiry, as a timestamp +// $now the moment of this pass +// $limit strikes before retirement +function vv_cert_apply(array $r, int $exp, int $now, int $limit): array { + $out = ['renewed' => false, 'failed' => false, 'retired' => false, 'from' => '', 'to' => '']; + + $r['checks'] = ($r['checks'] ?? 0) + 1; + $r['last_seen'] = $now; + // Cleared on sight: a domain that is back in NPM is not removed any more, whatever it was + // last pass. + $r['removed_at'] = null; + + // Both sides reduced to a date. last_expiry is stored as Y-m-d and NPM's expires_on carries a + // time, so comparing raw timestamps made every re-read of the same certificate look like a + // renewal to a few hours later. + $expDay = strtotime(date('Y-m-d', $exp)); + $prev = !empty($r['last_expiry']) ? strtotime($r['last_expiry']) : null; + + if ($prev !== null && $expDay > $prev) { + $r['renewals'] = ($r['renewals'] ?? 0) + 1; + $r['last_renewal'] = $now; + // A renewal clears the strikes and un-retires. The point of a strike count is "how long + // has this been broken", and it is no longer broken. + $r['strikes'] = 0; + $r['retired_at'] = null; + $out['renewed'] = true; + $out['from'] = date('Y-m-d', $prev); + $out['to'] = date('Y-m-d', $expDay); + } + + if ($exp < $now) { + $r['failures'] = ($r['failures'] ?? 0) + 1; + $r['strikes'] = ($r['strikes'] ?? 0) + 1; + $out['failed'] = true; + if ($r['strikes'] >= $limit && empty($r['retired_at'])) { + $r['retired_at'] = $now; + $r['retired_reason'] = "expired for {$r['strikes']} consecutive passes"; + $out['retired'] = true; + } + } + + $r['last_expiry'] = date('Y-m-d', $expDay); + $out['record'] = $r; + return $out; +} + +// ── One pass ────────────────────────────────────────────────────────────────── +$lockPath = sys_get_temp_dir() . '/vv_cert_history.lock'; +$lock = @fopen($lockPath, 'c'); +if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) { + echo "another pass is running\n"; + exit(0); +} + +try { + if ($miss = vv_auth_creds_missing('npm')) { echo "$miss\n"; exit(0); } + + $certs = vv_npm_list_certs(); + if (!$certs) { echo "NPM returned no certificates — nothing to record\n"; exit(0); } + + $hist = vv_cert_history_read(); + if (!$hist) { echo "cert_history.json is malformed — refusing to overwrite it\n"; exit(1); } + $store = $hist['domains'] ?? []; + $limit = vv_cert_strike_limit(); + $now = time(); + $seen = []; + $added = $renewed = $failed = $retired = $removed = 0; + $notes = []; + + // Resolved to one certificate per domain before anything is counted. A domain can appear on + // more than one certificate — three do here, left behind by re-issuing rather than replacing — + // and walking the list directly counted each of them as a separate check of the same domain, + // then read the second one's expiry as a renewal of the first. + // + // The winner is the latest expiry, because that is the one actually worth serving; the earliest + // created is kept as first_seen for the same reason NPM's created_on is used at all. + $byDomain = []; + foreach ($certs as $c) { + $exp = !empty($c['expires_on']) ? strtotime((string) $c['expires_on']) : false; + if ($exp === false) continue; + $created = !empty($c['created_on']) ? strtotime((string) $c['created_on']) : $now; + foreach (($c['domain_names'] ?? []) as $d) { + $d = strtolower(trim((string) $d)); + if ($d === '') continue; + if (!isset($byDomain[$d]) || $exp > $byDomain[$d]['exp']) + $byDomain[$d] = ['exp' => $exp, 'cert' => $c, 'created' => $created]; + else + $byDomain[$d]['created'] = min($byDomain[$d]['created'], $created ?: $now); + } + } + + { + foreach ($byDomain as $domain => $info) { + $exp = $info['exp']; + $c = $info['cert']; + $seen[$domain] = true; + + if (!isset($store[$domain])) { + $created = $info['created']; + $store[$domain] = [ + 'first_seen' => $created !== false ? $created : $now, + 'seeded_from' => 'npm_created_on', + 'checks' => 0, 'renewals' => 0, 'failures' => 0, 'strikes' => 0, + 'last_expiry' => null, 'last_renewal' => null, + 'retired_at' => null, 'removed_at' => null, + ]; + $added++; + $notes[] = "added $domain (first seen " . date('Y-m-d', $store[$domain]['first_seen']) . ')'; + } + + $r = &$store[$domain]; + $r['npm_id'] = $c['id'] ?? null; + $r['provider'] = $c['provider'] ?? null; + + $res = vv_cert_apply($r, $exp, $now, $limit); + $r = $res['record']; + if ($res['renewed']) { $renewed++; $notes[] = "renewed $domain ($res[from] → $res[to])"; } + if ($res['failed']) { $failed++; } + if ($res['retired']) { $retired++; $notes[] = "RETIRED $domain after {$r['strikes']} strikes"; } + unset($r); + } + } + + // Tracked but no longer in NPM. Marked, never struck and never deleted from the store — the + // history of a domain that used to exist is the reason this file is kept. + foreach ($store as $domain => &$r) { + if (isset($seen[$domain])) continue; + if (empty($r['removed_at'])) { + $r['removed_at'] = $now; + $removed++; + $notes[] = "no longer in NPM: $domain"; + } + } + unset($r); + + ksort($store); + $hist['domains'] = $store; + $hist['last_pass'] = $now; + + printf("%d certificates, %d domains tracked — added %d, renewed %d, failed %d, retired %d, removed %d\n", + count($certs), count($store), $added, $renewed, $failed, $retired, $removed); + foreach ($notes as $n) echo " $n\n"; + + if ($dryRun) { echo "dry run — nothing written\n"; exit(0); } + if (!vv_cert_history_write($hist)) { echo "could not write " . vv_cert_history_path() . "\n"; exit(1); } + echo 'wrote ' . vv_cert_history_path() . "\n"; + exit(0); + +} finally { + flock($lock, LOCK_UN); + fclose($lock); +} diff --git a/Plugin/unraid/Tools/cert_history.sh b/Plugin/unraid/Tools/cert_history.sh new file mode 100755 index 0000000..ca659bc --- /dev/null +++ b/Plugin/unraid/Tools/cert_history.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# ============================================================================================== +# ================================ Certificate History ========================================= +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Records what happens to every certificate NPM holds — first seen, renewals, failures, and how +# long each domain has been tracked — into DB_DIR/cert_history.json. The Certs tab reads it. +# +# NPM knows what a certificate is today and nothing about what it was, and cert_monitor.sh writes +# a snapshot the next run overwrites. Ten certificates on this host had been failing renewal for +# months without anything on any page being able to say so. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# A wrapper. The work is in cert_history.php, next to the NPM client it needs — the API token +# handling lives in include/auth.php and reimplementing it in bash to avoid a php call would be a +# second copy of the thing most worth having only one of. Same split as api_cache_writer and +# ai_repair_sweep. +# +# Counts start from zero on first run and are only ever observed. first_seen is seeded from NPM's +# own created_on, which is a real date; nothing else is back-filled. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# cert_history.sh one pass, updates the store +# cert_history.sh --dry-run reports what it would change, writes nothing +# cert_history.sh --status prints the store as a table +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# CERT_HISTORY_STRIKES consecutive failed passes before a domain is retired (default 5) +# DB_DIR cert_history.json is written here +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +php "$SCRIPT_DIR/cert_history.php" "$@" diff --git a/Plugin/unraid/api/cert.php b/Plugin/unraid/api/cert.php index a4ae61a..d796a5c 100644 --- a/Plugin/unraid/api/cert.php +++ b/Plugin/unraid/api/cert.php @@ -139,6 +139,82 @@ if ($action === 'npm') { exit; } +// ── Per-domain history, totals, and the DDNS containers ────────────────────── +// Read-only. Tools/cert_history.sh is what writes the store; serving it from here would mean the +// counters only advance when somebody happens to have the tab open. +if ($action === 'history') { + $file = rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/cert_history.json'; + $hist = is_file($file) ? (json_decode((string) @file_get_contents($file), true) ?: []) : []; + $doms = is_array($hist['domains'] ?? null) ? $hist['domains'] : []; + + $now = time(); + $rows = []; + $tot = ['tracked' => 0, 'active' => 0, 'retired' => 0, 'removed' => 0, + 'renewals' => 0, 'failures' => 0, 'checks' => 0, 'oldest' => null]; + + foreach ($doms as $domain => $r) { + $first = (int) ($r['first_seen'] ?? $now); + $retired = !empty($r['retired_at']); + $removed = !empty($r['removed_at']); + $rows[] = [ + 'domain' => (string) $domain, + 'first_seen'=> $first, + 'tracked' => vv_cert_span_php($first, $now), + 'checks' => (int) ($r['checks'] ?? 0), + 'renewals' => (int) ($r['renewals'] ?? 0), + 'failures' => (int) ($r['failures'] ?? 0), + 'strikes' => (int) ($r['strikes'] ?? 0), + 'expires' => $r['last_expiry'] ?? null, + 'last_renewal' => $r['last_renewal'] ?? null, + 'provider' => $r['provider'] ?? null, + 'state' => $removed ? 'removed' : ($retired ? 'retired' : 'active'), + ]; + $tot['tracked']++; + $tot[$removed ? 'removed' : ($retired ? 'retired' : 'active')]++; + $tot['renewals'] += (int) ($r['renewals'] ?? 0); + $tot['failures'] += (int) ($r['failures'] ?? 0); + $tot['checks'] += (int) ($r['checks'] ?? 0); + if ($tot['oldest'] === null || $first < $tot['oldest']) $tot['oldest'] = $first; + } + // Longest-tracked first: the domains with the most history are the ones the card is for. + usort($rows, fn($a, $b) => $a['first_seen'] <=> $b['first_seen']); + $tot['oldest_span'] = $tot['oldest'] ? vv_cert_span_php($tot['oldest'], $now) : '—'; + + // DDNS is on this tab because it is the other half of the same story: a certificate is issued + // against a name, and the name only points here while DDNS keeps it pointed. The ten dead + // certificates removed on 2026-08-15 all failed with NXDOMAIN. + $hostUp = strtoupper(vv_detect_host()); + $names = vv_parse_bash_array(vv_read_conf_raw(strtolower($hostUp) . '.conf'), + $hostUp . '_DDNS_CONTAINERS'); + $running = []; + foreach (vv_docker_containers() as $c) $running[$c['name']] = $c['status']; + $ddns = []; + foreach ($names as $n) { + $n = trim((string) $n); + if ($n === '') continue; + $ddns[] = ['name' => $n, + 'running' => isset($running[$n]), + 'status' => $running[$n] ?? 'not running']; + } + + echo json_encode(['ok' => true, 'rows' => $rows, 'totals' => $tot, 'ddns' => $ddns, + 'last_pass' => $hist['last_pass'] ?? null, + 'strike_limit' => (int) (vv_conf_vars()['CERT_HISTORY_STRIKES'] ?? 5)]); + exit; +} + +// Same shape as the tool's own formatter — years, months and days, because "3 years 6 months and +// 22 days" is how the question gets asked and 1298 days is the same fact nobody thinks in. +function vv_cert_span_php(int $from, int $to): string { + if ($to < $from) return '0d'; + $d = (new DateTime())->setTimestamp($from)->diff((new DateTime())->setTimestamp($to)); + $out = []; + if ($d->y) $out[] = $d->y . 'y'; + if ($d->m) $out[] = $d->m . 'mo'; + if ($d->d || !$out) $out[] = $d->d . 'd'; + return implode(' ', $out); +} + // ── Read configured domains (without running checks) ───────────────────────── if ($action === 'domains') { $hostId = vv_detect_host(); diff --git a/Plugin/unraid/pages/auth.php b/Plugin/unraid/pages/auth.php index d68b3b9..22253b3 100644 --- a/Plugin/unraid/pages/auth.php +++ b/Plugin/unraid/pages/auth.php @@ -243,6 +243,32 @@ require_once dirname(__DIR__) . '/include/ai_chat.php'; .vv-au-cert-days { font-size:28px;font-weight:700;line-height:1;margin:6px 0 2px; } .vv-au-cert-bar { height:3px;border-radius:2px;background:#1a1a1a;overflow:hidden;margin-top:8px; } .vv-au-cert-fill { height:100%;border-radius:2px;transition:width .3s; } + +/* ── Cert history ────────────────────────────────────────────────────────── */ +.vv-au-tot { display:grid;grid-template-columns:repeat(auto-fit,minmax(96px,1fr));gap:8px; } +.vv-au-tot-b { background:#161616;border:1px solid #222;border-radius:5px;padding:8px 10px;text-align:center; } +.vv-au-tot-n { font-size:19px;font-weight:700;color:#bbb;line-height:1.1; } +.vv-au-tot-n.bad { color:#ef5350; } +.vv-au-tot-l { font-size:9px;color:#444;text-transform:uppercase;letter-spacing:.05em;margin-top:2px; } +/* Fixed columns rather than flex: forty rows of counters only read as a table if the numbers line + up down the page, and a domain name is the one part whose width varies. */ +.vv-au-hist-row { display:grid;grid-template-columns:1fr 34px 34px 74px 84px auto;gap:6px; + align-items:center;padding:4px 12px;border-bottom:1px solid #1a1a1a;font-size:11px; } +.vv-au-hist-row:last-child { border-bottom:none; } +.vv-au-hist-row:hover { background:#141414; } +.vv-au-hist-row.retired { background:#150c0c; } +.vv-au-hist-row.removed { opacity:.45; } +.vv-au-hist-dom { color:#bbb;font-family:monospace;overflow:hidden;text-overflow:ellipsis;white-space:nowrap; } +.vv-au-hist-n { text-align:right;font-weight:600;font-size:11px; } +.vv-au-hist-n.ok { color:#4caf50; } +.vv-au-hist-n.bad{ color:#ef5350; } +.vv-au-hist-n.dim{ color:#333; } +.vv-au-hist-t { color:#666;font-size:10px;text-align:right; } +.vv-au-hist-e { color:#444;font-size:10px;font-family:monospace;text-align:right; } +.vv-au-hist-note { font-size:9px;color:#3a3a3a;font-weight:normal;text-transform:none;letter-spacing:0; } +.vv-au-strike { font-size:9px;color:#ff9800;background:#1f1200;border:1px solid #3a2800; + border-radius:2px;padding:0 5px;white-space:nowrap; } +.vv-au-dot { width:7px;height:7px;border-radius:50%;flex-shrink:0;display:inline-block; } 'Proxies', 'users' => 'Users & Groups',
Loading…
+ + +
+
+
+
+ Domain history + +
+
Loading…
+
+
+
DDNS
+
Loading…
+
+
@@ -1769,9 +1813,84 @@ function _renderNpmCerts(data) { }).join(''); } +// ── History, totals and DDNS ───────────────────────────────────────────────── +// Read-only: the counters advance when Tools/cert_history.sh runs, not when this page is opened. +// A page that wrote the history it displays would count a refresh as an observation. +function _loadCertHistory() { + const list = document.getElementById('vv-au-hist-list'); + const tot = document.getElementById('vv-au-hist-totals'); + const ddns = document.getElementById('vv-au-ddns'); + if (!list) return; + fetch(CERT_API + '?action=history') + .then(r => r.json()) + .then(d => { + if (!d.ok) throw new Error(d.error || 'history unavailable'); + + const t = d.totals || {}; + if (!t.tracked) { + tot.innerHTML = ''; + list.innerHTML = '
No history yet — run ' + + 'Tools/cert_history.sh once to start tracking.
'; + ddns.innerHTML = ''; + return; + } + + // Failures are only red when there are any: a zero in an alarm colour trains you to ignore + // the colour rather than the number. + tot.innerHTML = `
+ ${_totBox(t.tracked, 'domains tracked')} + ${_totBox(t.active, 'active')} + ${_totBox(t.renewals, 'renewals seen')} + ${_totBox(t.failures, 'failures', t.failures ? 'bad' : '')} + ${_totBox(t.retired, 'retired', t.retired ? 'bad' : '')} + ${_totBox(t.checks, 'checks')} + ${_totBox(t.oldest_span || '—', 'longest tracked')} +
`; + + const when = document.getElementById('vv-au-hist-when'); + if (when) when.textContent = d.last_pass + ? 'last pass ' + new Date(d.last_pass * 1000).toLocaleString() + : 'never run'; + + list.innerHTML = (d.rows || []).map(r => { + const cls = r.state === 'retired' ? ' retired' : (r.state === 'removed' ? ' removed' : ''); + // Strikes are only worth showing while they are accruing; a retired domain already says so. + const strike = (r.strikes && r.state === 'active') + ? `${r.strikes}/${d.strike_limit} strikes` : ''; + const tag = r.state === 'retired' ? 'retired' + : r.state === 'removed' ? 'removed' : ''; + return `
+ ${_esc(r.domain)} + ${r.renewals} + ${r.failures} + ${_esc(r.tracked)} + ${_esc(r.expires || '—')} + ${strike}${tag} +
`; + }).join('') || '
Nothing tracked yet.
'; + + ddns.innerHTML = (d.ddns || []).length + ? d.ddns.map(c => `
+ + ${_esc(c.name)} + ${_esc(c.status)} +
`).join('') + : '
No DDNS containers configured.
'; + }) + .catch(e => { + list.innerHTML = `
${_esc(e.message || 'Failed')}
`; + }); +} + +function _totBox(n, label, cls) { + return `
${_esc(String(n))}
` + + `
${_esc(label)}
`; +} + function _loadCerts(onDone) { const grid = document.getElementById('vv-au-cert-grid'); grid.innerHTML = '
Loading…
'; + _loadCertHistory(); fetch(CERT_API + '?action=npm') .then(r => r.json()) .then(d => { diff --git a/Plugin/unraid/pages/readme/ui-map.md b/Plugin/unraid/pages/readme/ui-map.md index f0e3185..633cc6d 100644 --- a/Plugin/unraid/pages/readme/ui-map.md +++ b/Plugin/unraid/pages/readme/ui-map.md @@ -1196,6 +1196,16 @@ Saved into `master.conf`, which does not need to be opened by hand. | `BUG_REPORT_LOCAL_ENABLED` | a switch | in this section | LOCAL ON — reports go to your own Gitea (see HOSTN_BUG_REPORT_* in host*.conf) and stay there. They do NOT reach the Varaverk maintainer. Turn it on if you want your own backlog. LOCAL OFF — reports open a prefilled GitHub issue you submit under your own account. | | `BUG_REPORT_GITHUB_REPO` | a text box | in this section | — | +## Certificate History + +Route: Settings tab → All settings → *Certificate History* + +Saved into `master.conf`, which does not need to be opened by hand. + +| Setting | Control | Where | What it does | +|---|---|---|---| +| `CERT_HISTORY_STRIKES` | a number box | in this section | expired passes before a domain is retired from the active list | + ## Certificate Monitor Reachable from: