619 lines
32 KiB
PHP
619 lines
32 KiB
PHP
<?php
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
// PURPOSE
|
|
// Watchdog page data layer. Collects what every watchdog has recorded — resource strikes,
|
|
// container restart history and the failed-container skip list, system strikes and OOM
|
|
// count, storage growth and log strikes, network/NPM strikes, and the reboot log — for
|
|
// this host and each partner.
|
|
//
|
|
// DESIGN PRINCIPLES
|
|
// Reads the strike counters; never resets them.
|
|
// Strike state belongs to the watchdog that owns it. A page that cleared strikes would
|
|
// silently undo an escalation the watchdog was deliberately building toward.
|
|
//
|
|
// One state file per watchdog, parsed independently.
|
|
// resource / container / system / storage / network each keep their own db. A watchdog
|
|
// that has never run leaves its file absent, which is a distinct and meaningful state.
|
|
//
|
|
// The skip list is surfaced, not hidden.
|
|
// docker_watchdog_failed.db records containers the docker watchdog has given up on.
|
|
// Those are exactly the ones an operator needs to see, so they are shown rather than
|
|
// filtered out of the healthy-looking list.
|
|
//
|
|
// OPERATIONAL SAFEGUARDS
|
|
// Every state read is suppressed and defaulted.
|
|
// @file_get_contents with a ?: fallback throughout — an absent or unreadable db yields
|
|
// empty/zero and that watchdog's card renders as "no data", never as a fatal.
|
|
//
|
|
// Absent counters read as zero, not as unknown-therefore-alarming.
|
|
// A watchdog that has not yet written state is reported quiet rather than as a
|
|
// problem, so a fresh boot does not light up the page with false strikes.
|
|
//
|
|
// Remote collection failures are per-node.
|
|
// One unreachable partner drops that node's card; the local host and every other
|
|
// partner still render.
|
|
//
|
|
// Read-only. Nothing here restarts a container, clears a strike, or triggers a reboot.
|
|
//
|
|
// EXPORTS
|
|
// Local vv_wd_local_system(), vv_wd_local_states()
|
|
// Remote vv_wd_remote_data(), vv_wd_node_config()
|
|
// Assembly vv_wd_all()
|
|
// Parsing vv_wd_bash_array(), vv_wd_bash_assoc(), vv_wd_scalar(), vv_wd_parse_kv(),
|
|
// vv_wd_parse_restart_log(), vv_wd_parse_skiplist(), vv_wd_parse_reboot_log(),
|
|
// vv_wd_parse_storage_state(), vv_wd_parse_network_state()
|
|
//
|
|
// CONFIGURATION
|
|
// STATE_DIR resource_watchdog_state.db, container_watchdog_state.db,
|
|
// docker_watchdog_failed.db, system_watchdog_state.db,
|
|
// system_watchdog_oom.db, storage/network watchdog state
|
|
// RW_CRITICAL_CONTAINERS containers the resource watchdog treats as critical
|
|
// HOST*_SSH_KEY used to collect partner watchdog state
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
require_once __DIR__ . '/common.php';
|
|
require_once __DIR__ . '/partnership.php';
|
|
|
|
// Watchdog tab data helpers
|
|
|
|
// Rows in system_watchdog_state.db that are bookkeeping, not strikes. The file is a flat key/value
|
|
// store shared by the counters and the watchdog's own liveness record, so the only thing separating
|
|
// the two is knowing which is which — there is no marker in the format.
|
|
//
|
|
// watchdog_cycle is a unix timestamp, written every cycle by stability_watchdog.sh:883 for
|
|
// docker_watchdog's stale-state guard. Published as a strike it reads as 1.7 billion of them.
|
|
//
|
|
// Add to this list, never to a shape test: a strike count and a timestamp are both positive
|
|
// integers, and a threshold like "bigger than a billion means it is a clock" is a rule that works
|
|
// until it doesn't and fails silently in the direction of hiding a real strike.
|
|
const VV_WD_SYS_BOOKKEEPING = ['watchdog_cycle'];
|
|
|
|
// Every container that exists on this host, running or not, for deciding whether a name in a
|
|
// watchdog list still refers to anything.
|
|
//
|
|
// `docker ps -a` rather than vv_docker_containers() + vv_docker_stopped(): those two are filtered
|
|
// by status and between them still miss states like restarting, and a container this page called
|
|
// "not installed" because it happened to be mid-restart would be worse than not checking at all.
|
|
// The whole point of the badge is that it is only ever shown when it is certainly true.
|
|
//
|
|
// Returns an empty array when docker cannot be reached, and callers must treat that as "unknown"
|
|
// rather than "nothing is installed" — otherwise a docker outage marks every entry dead.
|
|
function vv_wd_installed_containers(): array {
|
|
@exec("docker ps -a --format '{{.Names}}' 2>/dev/null", $out, $rc);
|
|
if ($rc !== 0) return [];
|
|
return array_values(array_filter(array_map('trim', $out), fn($n) => $n !== ''));
|
|
}
|
|
|
|
// Active timed mutes, as [name => ['left' => seconds, 'reason' => text]]. Expiry is a read-time
|
|
// comparison here for the same reason it is in wd_mute_active(): a mute ends when it says it does,
|
|
// whether or not anything has pruned the file since.
|
|
//
|
|
// Surfaced because a suppression nobody can see is the problem this feature exists to fix. An
|
|
// invisible exemption gets forgotten exactly like a permanent one — the only difference would be
|
|
// that this one also lies about how long it lasts.
|
|
function vv_wd_mutes(string $file = ''): array {
|
|
$path = $file ?: (vv_conf_vars()['WATCHDOG_MUTE_FILE'] ?? STATE_DIR . '/watchdog_mutes.db');
|
|
$raw = @file_get_contents($path);
|
|
if ($raw === false) return [];
|
|
$now = time();
|
|
$out = [];
|
|
foreach (explode("\n", $raw) as $line) {
|
|
$line = trim($line);
|
|
if ($line === '') continue;
|
|
$p = explode('|', $line);
|
|
if (count($p) < 2) continue;
|
|
$until = (int)$p[1];
|
|
if ($until <= $now || $p[0] === '') continue;
|
|
$out[$p[0]] = ['left' => $until - $now, 'reason' => trim($p[2] ?? '')];
|
|
}
|
|
return $out;
|
|
}
|
|
|
|
// How many minutes are supposed to pass between watchdog cycles.
|
|
//
|
|
// Parsed from varaverk.cron, which the scheduler regenerates at array start and is the only place
|
|
// the answer exists — the interval is a schedule, not a conf key. Reading it rather than assuming
|
|
// 15 means that changing the schedule also moves the point at which the page calls a cycle
|
|
// overdue, instead of leaving a constant here to drift out of agreement with reality.
|
|
//
|
|
// Falls back to 15 when the cron is missing or the entry is written in a form this does not
|
|
// recognise. A wrong-but-sane interval produces a slightly early or late warning; refusing to
|
|
// answer would remove the liveness check altogether, which is the failure being fixed.
|
|
function vv_wd_cycle_interval_min(): int {
|
|
$cron = @file_get_contents('/boot/config/plugins/varaverk/varaverk.cron') ?: '';
|
|
foreach (explode("\n", $cron) as $line) {
|
|
if (!str_contains($line, 'watchdog_orchestrator')) continue;
|
|
if (preg_match('#^\s*\*/(\d+)\s#', $line, $m)) return max(1, (int)$m[1]);
|
|
}
|
|
return 15;
|
|
}
|
|
|
|
// ── Conf array parser (bash arrays) ──────────────────────────────────────────
|
|
|
|
function vv_wd_bash_array(string $raw, string $varname): array {
|
|
return vv_parse_bash_array($raw, $varname);
|
|
}
|
|
|
|
function vv_wd_bash_assoc(string $raw, string $varname): array {
|
|
// declare -A VARNAME=( ["key"]=val ["key2"]=val2 )
|
|
if (!preg_match('/^\s*declare\s+-A\s+' . preg_quote($varname, '/') . '\s*=\s*\(\s*(.*?)\s*\)/ms', $raw, $m))
|
|
return [];
|
|
preg_match_all('/\["([^"]+)"\]\s*=\s*"?([^"\s\)]*)"?/', $m[1], $pairs);
|
|
$out = [];
|
|
foreach ($pairs[1] as $i => $k) $out[$k] = $pairs[2][$i];
|
|
return $out;
|
|
}
|
|
|
|
function vv_wd_scalar(string $raw, string $varname): string {
|
|
return vv_parse_conf_scalar($raw, $varname);
|
|
}
|
|
|
|
// ── State file parsers ────────────────────────────────────────────────────────
|
|
|
|
// Parses key:value format (with optional key=value mixed in)
|
|
function vv_wd_parse_kv(string $text): array {
|
|
$out = [];
|
|
foreach (explode("\n", $text) as $line) {
|
|
$line = trim($line);
|
|
if (!$line) continue;
|
|
if (str_contains($line, ':')) {
|
|
[$k, $v] = explode(':', $line, 2);
|
|
$out[trim($k)] = trim($v);
|
|
} elseif (str_contains($line, '=')) {
|
|
[$k, $v] = explode('=', $line, 2);
|
|
$out[trim($k)] = trim($v, '"\'');
|
|
}
|
|
}
|
|
return $out;
|
|
}
|
|
|
|
// Restart log: "container|2026-08-02 13:15:07" one per line.
|
|
//
|
|
// The second field is a formatted local timestamp, not an epoch — docker_watchdog.sh writes it
|
|
// with date '+%Y-%m-%d %H:%M:%S' because its own rolling-window trim compares the strings
|
|
// lexically in awk, which is correct for that format. This function used to cast it with (int),
|
|
// which stops at the first non-digit and yielded 2026 for every line ever written. 2026 is below
|
|
// any plausible cutoff, so every entry was discarded and both restart panels — the Monitor card
|
|
// and the Watchdog page — were permanently empty. Not visibly broken: an empty list reads
|
|
// exactly like "nothing has restarted", which is the answer you least want to be wrong about
|
|
// during a restart loop.
|
|
//
|
|
// Parsed, not reformatted. Changing what the watchdog writes would strand every existing entry
|
|
// and every awk comparison in Tools/watchdog_skip_list_manager.sh.
|
|
function vv_wd_parse_restart_log(string $text, int $windowSeconds = 86400): array {
|
|
$cutoff = time() - $windowSeconds;
|
|
$entries = [];
|
|
foreach (explode("\n", trim($text)) as $line) {
|
|
$line = trim($line);
|
|
if (!$line || !str_contains($line, '|')) continue;
|
|
[$name, $raw] = explode('|', $line, 2);
|
|
$ts = vv_wd_restart_ts(trim($raw));
|
|
if ($ts >= $cutoff) $entries[] = ['name' => trim($name), 'ts' => $ts];
|
|
}
|
|
usort($entries, fn($a, $b) => $b['ts'] - $a['ts']);
|
|
return $entries;
|
|
}
|
|
|
|
// vv_wd_restart_ts() lives in common.php — include/monitor.php parses the same file and does not
|
|
// include this one, so a helper defined here would be a fatal on the dashboard.
|
|
|
|
// Skip list: one container name per line
|
|
function vv_wd_parse_skiplist(string $text): array {
|
|
return array_values(array_filter(array_map('trim', explode("\n", $text))));
|
|
}
|
|
|
|
// Reboot log: one timestamp per line
|
|
function vv_wd_parse_reboot_log(string $text, int $windowHrs = 12): array {
|
|
$cutoff = time() - ($windowHrs * 3600);
|
|
$entries = [];
|
|
foreach (explode("\n", trim($text)) as $line) {
|
|
$ts = (int)trim($line);
|
|
if ($ts > 0 && $ts >= $cutoff) $entries[] = $ts;
|
|
}
|
|
rsort($entries);
|
|
return $entries;
|
|
}
|
|
|
|
// ── Local system snapshot ─────────────────────────────────────────────────────
|
|
|
|
function vv_wd_local_system(): array {
|
|
$sys = vv_system_info();
|
|
$mem = vv_memory_breakdown();
|
|
$loadRaw = @file_get_contents('/proc/loadavg') ?: '0';
|
|
$daemonOk = (trim(shell_exec('docker info >/dev/null 2>&1; echo $?') ?: '1') === '0');
|
|
$oomCount = (int)trim(@file_get_contents(STATE_DIR . '/system_watchdog_oom.db') ?: '0');
|
|
// nproc first, and cpu_cores only as a fallback — the reverse of what this used to do.
|
|
//
|
|
// The page draws its load bar and colours against this number multiplied by the same
|
|
// RW_LOAD_* multipliers resource_watchdog.sh uses, and that script computes its thresholds
|
|
// from `nproc` (resource_watchdog.sh:194). cpu_cores is the API's physical core count: 16 here
|
|
// against 32 threads. Preferring it made the page turn amber at load 32 and red at 48 while
|
|
// the watchdog did not reach soft pressure until 64 and medium until 96 — the page reporting
|
|
// a crisis about a subsystem that considered the machine idle, which is the one thing a page
|
|
// whose entire job is showing what the watchdogs think must not do.
|
|
//
|
|
// Whether load should be judged against threads or cores is a real argument; it is not this
|
|
// file's argument to have. The number here has to be the number the actor used, or the page
|
|
// is describing a different machine.
|
|
$cores = (int)(trim(shell_exec('nproc 2>/dev/null') ?: '') ?: ($sys['cpu_cores'] ?: 1));
|
|
return [
|
|
'mem_total' => (int)($mem['total_kb'] * 1024),
|
|
'mem_avail' => (int)($mem['free_kb'] * 1024),
|
|
'load1' => (float)explode(' ', trim($loadRaw))[0],
|
|
'cores' => $cores,
|
|
'uptime' => $sys['uptime_sec'] ?? 0,
|
|
'daemon_ok' => $daemonOk,
|
|
'oom_count' => $oomCount,
|
|
];
|
|
}
|
|
|
|
// ── Storage / network state parsers ──────────────────────────────────────────
|
|
|
|
function vv_wd_parse_storage_state(string $raw): array {
|
|
$growth = []; $log = [];
|
|
foreach (explode("\n", $raw) as $line) {
|
|
$line = trim($line);
|
|
if (!$line || !str_contains($line, ':')) continue;
|
|
[$k, $v] = explode(':', $line, 2);
|
|
$count = (int)trim($v);
|
|
if ($count <= 0) continue;
|
|
$key = trim($k);
|
|
if (str_starts_with($key, 'appdata_growth_'))
|
|
$growth[substr($key, strlen('appdata_growth_'))] = $count;
|
|
elseif (str_starts_with($key, 'appdata_log_'))
|
|
$log[substr($key, strlen('appdata_log_'))] = $count;
|
|
}
|
|
return ['growth_strikes' => $growth, 'log_strikes' => $log];
|
|
}
|
|
|
|
function vv_wd_parse_network_state(string $raw): array {
|
|
$npm = 0;
|
|
foreach (explode("\n", $raw) as $line) {
|
|
$line = trim($line);
|
|
if (str_starts_with($line, 'npm:')) $npm = (int)trim(substr($line, 4));
|
|
}
|
|
return ['npm_strikes' => $npm];
|
|
}
|
|
|
|
// ── Local state files ─────────────────────────────────────────────────────────
|
|
|
|
function vv_wd_local_states(string $restartLogPath): array {
|
|
$rwRaw = @file_get_contents(STATE_DIR . '/resource_watchdog_state.db') ?: '';
|
|
$dockRaw = @file_get_contents(STATE_DIR . '/container_watchdog_state.db') ?: '';
|
|
$skipRaw = @file_get_contents(STATE_DIR . '/docker_watchdog_failed.db') ?: '';
|
|
$sysRaw = @file_get_contents(STATE_DIR . '/system_watchdog_state.db') ?: '';
|
|
$rebootRaw = @file_get_contents(STATE_DIR . '/system_watchdog_reboots.db') ?: '';
|
|
$restartRaw= @file_get_contents($restartLogPath) ?: '';
|
|
$storRaw = @file_get_contents(STATE_DIR . '/storage_watchdog_state.db') ?: '';
|
|
$netWdRaw = @file_get_contents(STATE_DIR . '/network_watchdog_state.db') ?: '';
|
|
|
|
$rw = vv_wd_parse_kv($rwRaw);
|
|
$dock = vv_wd_parse_kv($dockRaw);
|
|
$sys = vv_wd_parse_kv($sysRaw);
|
|
|
|
// Container strikes: everything in docker state that isn't a flag
|
|
$strikes = [];
|
|
foreach ($dock as $k => $v) {
|
|
if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0)
|
|
$strikes[$k] = (int)$v;
|
|
}
|
|
|
|
// Stability strikes: everything in sys state that is actually a strike.
|
|
//
|
|
// The old guard was `!str_contains($k, '=')`, written on the belief that the bookkeeping rows
|
|
// in system_watchdog_state.db keep their `=` inside the key because the file mixes separators
|
|
// — `ram:0` and `sshd:0` beside `kernel_oops_count=0` and `watchdog_cycle=...`. They do not:
|
|
// vv_wd_parse_kv() splits on both, so the guard never matched anything and every bookkeeping
|
|
// row was published as a strike. watchdog_cycle is a unix timestamp written once per cycle by
|
|
// stability_watchdog.sh:883 as a liveness heartbeat, so the Stability card was drawing
|
|
// "watchdog cycle — 1786716931" next to a limit of 2.
|
|
$sysStrikes = [];
|
|
foreach ($sys as $k => $v) {
|
|
if (in_array($k, VV_WD_SYS_BOOKKEEPING, true)) continue;
|
|
if ((int)$v > 0) $sysStrikes[$k] = (int)$v;
|
|
}
|
|
|
|
// Growth baseline info (container count + age in seconds)
|
|
$growthFile = STATE_DIR . '/watchdog_appdata_growth.db';
|
|
$baselineCount = file_exists($growthFile) ? max(0, count(file($growthFile)) - 0) : 0;
|
|
$baselineAgeSec = file_exists($growthFile) ? time() - (int)filemtime($growthFile) : null;
|
|
|
|
return [
|
|
'rw_level' => (int)($rw['rm_action_level'] ?? 0),
|
|
'rw_recover' => (int)($rw['rm_recover_cycles'] ?? 0),
|
|
'rw_paused' => array_filter(explode(',', $rw['rm_paused_containers'] ?? '')),
|
|
'rw_stopped' => array_filter(explode(',', $rw['rm_stopped_containers'] ?? '')),
|
|
'mem_shutdown' => ($rw['mem_shutdown_active'] ?? 'false') === 'true',
|
|
'daemon_strikes' => (int)($dock['daemon_strikes'] ?? 0),
|
|
'daemon_restart' => ($dock['daemon_restarted_flag'] ?? 'false') === 'true',
|
|
'ctr_strikes' => $strikes,
|
|
'skiplist' => vv_wd_parse_skiplist($skipRaw),
|
|
'sys_strikes' => $sysStrikes,
|
|
// The heartbeat, not a strike — see VV_WD_SYS_BOOKKEEPING. stability_watchdog.sh writes it
|
|
// last in the chain, so a fresh value means a whole cycle completed rather than started.
|
|
'last_cycle' => (int)($sys['watchdog_cycle'] ?? 0),
|
|
'mutes' => vv_wd_mutes(),
|
|
'reboots' => vv_wd_parse_reboot_log($rebootRaw),
|
|
'restarts' => vv_wd_parse_restart_log($restartRaw),
|
|
'storage_wd' => vv_wd_parse_storage_state($storRaw) + [
|
|
'baseline_count' => $baselineCount,
|
|
'baseline_age_sec' => $baselineAgeSec,
|
|
],
|
|
'network_wd' => vv_wd_parse_network_state($netWdRaw),
|
|
];
|
|
}
|
|
|
|
// ── Remote data via SSH ───────────────────────────────────────────────────────
|
|
|
|
function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath): array {
|
|
// Bundle into one SSH call.
|
|
// /proc/meminfo is passed as a raw section (not awk-parsed) to avoid quoting
|
|
// fragility — escapeshellarg() single-quotes the whole command so awk \$2
|
|
// inside double-quotes is unreliable across Unraid builds.
|
|
// State files live under the REMOTE's own SCRIPTS_DIR (may differ from ours in flash mode),
|
|
// so it is resolved on the far side — same idiom as vv_remote_state_cmd().
|
|
//
|
|
// The layout is probed rather than assumed. State moved from SCRIPTS_DIR/State_Files to
|
|
// DATA_DIR/state, and the histories from DATA_DIR's root into DATA_DIR/db, but a partner is
|
|
// not guaranteed to have pulled that yet — and this is the call that reports whether the
|
|
// partner's watchdogs are healthy. Guessing wrong returns empty strings for every state
|
|
// file, which reads as "partner has no strikes" rather than as an error. Probing costs one
|
|
// directory test and makes the answer correct in both directions, which also means the two
|
|
// hosts can be upgraded in either order.
|
|
$restartLogName = basename($restartLogPath);
|
|
$cmd = 'sd=$(grep -m1 SCRIPTS_DIR= /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null'
|
|
. ' | cut -d\'"\' -f2); sd="${sd:-/boot/config/plugins/varaverk}"; '
|
|
. 'sf="$sd/data/state"; [ -d "$sf" ] || sf="$sd/State_Files"; '
|
|
. 'db="$sd/data/db"; [ -d "$db" ] || db="$sd/data"; '
|
|
. "printf 'UPTIME:%s\nLOAD:%s\nCORES:%s\nDAEMON:%s\nOOM:%s\nBASELINECOUNT:%s\nBASELINEAGE:%s\n---MEMINFO---\n%s\n---RW---\n%s\n---DOCK---\n%s\n---SKIP---\n%s\n---SYS---\n%s\n---REBOOT---\n%s\n---RESTART---\n%s\n---STORAGE---\n%s\n---NETWORK---\n%s\n' "
|
|
// awk, not `cut -d" "`. This is a PHP SINGLE-quoted string, which does not process \" —
|
|
// so the shell received a literal backslash-quote, cut read the quote as a FILENAME
|
|
// ("cut: '\"': No such file or directory"), and UPTIME and LOAD came back empty while every
|
|
// quote-free field beside them parsed fine. awk needs no delimiter argument at all.
|
|
. '"$(awk \'{print $1}\' /proc/uptime)" '
|
|
. '"$(awk \'{print $1}\' /proc/loadavg)" '
|
|
. '"$(nproc)" '
|
|
. '"$(docker info >/dev/null 2>&1 && echo ok || echo err)" '
|
|
. '"$(cat "$sf/system_watchdog_oom.db" 2>/dev/null||echo 0)" '
|
|
. '"$(wc -l < "$sf/watchdog_appdata_growth.db" 2>/dev/null||echo 0)" '
|
|
. '"$(stat -c %Y "$sf/watchdog_appdata_growth.db" 2>/dev/null||echo 0)" '
|
|
. '"$(cat /proc/meminfo 2>/dev/null)" '
|
|
. '"$(cat "$sf/resource_watchdog_state.db" 2>/dev/null)" '
|
|
. '"$(cat "$sf/container_watchdog_state.db" 2>/dev/null)" '
|
|
. '"$(cat "$sf/docker_watchdog_failed.db" 2>/dev/null)" '
|
|
. '"$(cat "$sf/system_watchdog_state.db" 2>/dev/null)" '
|
|
. '"$(cat "$sf/system_watchdog_reboots.db" 2>/dev/null)" '
|
|
. '"$(cat "$db/' . $restartLogName . '" 2>/dev/null)" '
|
|
. '"$(cat "$sf/storage_watchdog_state.db" 2>/dev/null)" '
|
|
. '"$(cat "$sf/network_watchdog_state.db" 2>/dev/null)"';
|
|
|
|
$out = vv_pt_ssh($ip, $sshKey, $cmd, 8);
|
|
if (!$out) return null;
|
|
|
|
// Parse sections
|
|
$sections = preg_split('/^---\w+---$/m', $out);
|
|
$header = $sections[0] ?? '';
|
|
$memInfoRaw = $sections[1] ?? '';
|
|
$rwRaw = $sections[2] ?? '';
|
|
$dockRaw = $sections[3] ?? '';
|
|
$skipRaw = $sections[4] ?? '';
|
|
$sysRaw = $sections[5] ?? '';
|
|
$rebootRaw = $sections[6] ?? '';
|
|
$restartRaw = $sections[7] ?? '';
|
|
$storRaw = $sections[8] ?? '';
|
|
$netWdRaw = $sections[9] ?? '';
|
|
|
|
// Parse header lines
|
|
$hdr = [];
|
|
foreach (explode("\n", $header) as $line) {
|
|
if (preg_match('/^(\w+):(.*)$/', trim($line), $m)) $hdr[$m[1]] = trim($m[2]);
|
|
}
|
|
|
|
// Parse /proc/meminfo section — no awk, no quoting issues
|
|
$memTotal = 0; $memAvail = 0;
|
|
if (preg_match('/^MemTotal:\s+(\d+)/m', $memInfoRaw, $m)) $memTotal = (int)$m[1] * 1024;
|
|
if (preg_match('/^MemAvailable:\s+(\d+)/m', $memInfoRaw, $m)) $memAvail = (int)$m[1] * 1024;
|
|
|
|
$rw = vv_wd_parse_kv($rwRaw);
|
|
$dock = vv_wd_parse_kv($dockRaw);
|
|
$sys = vv_wd_parse_kv($sysRaw);
|
|
|
|
$strikes = [];
|
|
foreach ($dock as $k => $v) {
|
|
if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0)
|
|
$strikes[$k] = (int)$v;
|
|
}
|
|
// Same exclusion as the local path above — a partner's heartbeat is no more a strike than
|
|
// this host's. The two builders are maintained together.
|
|
$sysStrikes = [];
|
|
foreach ($sys as $k => $v) {
|
|
if (in_array($k, VV_WD_SYS_BOOKKEEPING, true)) continue;
|
|
if ((int)$v > 0) $sysStrikes[$k] = (int)$v;
|
|
}
|
|
|
|
$baselineCount = (int)($hdr['BASELINECOUNT'] ?? 0);
|
|
$baselineTs = (int)($hdr['BASELINEAGE'] ?? 0);
|
|
$baselineAgeSec = $baselineTs > 0 ? time() - $baselineTs : null;
|
|
|
|
return [
|
|
'system' => [
|
|
'mem_total' => $memTotal,
|
|
'mem_avail' => $memAvail,
|
|
'load1' => (float)($hdr['LOAD'] ?? 0),
|
|
'cores' => (int)($hdr['CORES'] ?? 1),
|
|
'uptime' => (int)($hdr['UPTIME'] ?? 0),
|
|
'daemon_ok' => ($hdr['DAEMON'] ?? '') === 'ok',
|
|
'oom_count' => (int)($hdr['OOM'] ?? 0),
|
|
],
|
|
'states' => [
|
|
'rw_level' => (int)($rw['rm_action_level'] ?? 0),
|
|
'rw_recover' => (int)($rw['rm_recover_cycles'] ?? 0),
|
|
'rw_paused' => array_filter(explode(',', $rw['rm_paused_containers'] ?? '')),
|
|
'rw_stopped' => array_filter(explode(',', $rw['rm_stopped_containers'] ?? '')),
|
|
'mem_shutdown' => ($rw['mem_shutdown_active'] ?? 'false') === 'true',
|
|
'daemon_strikes'=> (int)($dock['daemon_strikes'] ?? 0),
|
|
'daemon_restart'=> ($dock['daemon_restarted_flag'] ?? 'false') === 'true',
|
|
'ctr_strikes' => $strikes,
|
|
'skiplist' => vv_wd_parse_skiplist($skipRaw),
|
|
'sys_strikes' => $sysStrikes,
|
|
'last_cycle' => (int)($sys['watchdog_cycle'] ?? 0),
|
|
'reboots' => vv_wd_parse_reboot_log($rebootRaw),
|
|
'restarts' => vv_wd_parse_restart_log($restartRaw),
|
|
'storage_wd' => vv_wd_parse_storage_state($storRaw) + [
|
|
'baseline_count' => $baselineCount,
|
|
'baseline_age_sec' => $baselineAgeSec,
|
|
],
|
|
'network_wd' => vv_wd_parse_network_state($netWdRaw),
|
|
],
|
|
];
|
|
}
|
|
|
|
// ── Config inventory ──────────────────────────────────────────────────────────
|
|
|
|
function vv_wd_node_config(string $slot, string $raw, string $masterRaw): array {
|
|
$id = strtoupper($slot);
|
|
return [
|
|
'monitored' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINERS"),
|
|
'urls' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINER_URLS"),
|
|
'required' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_REQUIRED_CONTAINERS"),
|
|
'ignore' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_SCAN_IGNORE"),
|
|
'pause_list' => vv_wd_bash_array($raw, "{$id}_RW_PAUSE_CONTAINERS"),
|
|
'stop_list' => vv_wd_bash_array($raw, "{$id}_RW_STOP_CONTAINERS"),
|
|
'critical' => vv_wd_bash_array($masterRaw, "RW_CRITICAL_CONTAINERS"),
|
|
// Network watchdog — host-specific
|
|
'ddns_domain' => vv_wd_scalar($raw, "{$id}_NETWORK_WATCHDOG_DDNS_DOMAIN"),
|
|
'ddns_container' => vv_wd_scalar($raw, "{$id}_NETWORK_WATCHDOG_DDNS_CONTAINER"),
|
|
'npm_url' => vv_wd_scalar($raw, "{$id}_NETWORK_WATCHDOG_NPM_URL"),
|
|
// Storage watchdog — host-specific appdata suppress ceilings
|
|
'appdata_sizes' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_APPDATA_SIZES"),
|
|
];
|
|
}
|
|
|
|
// ── Entry point ───────────────────────────────────────────────────────────────
|
|
|
|
function vv_wd_all(): array {
|
|
$currentHost = vv_detect_host();
|
|
$tsPeers = vv_pt_ts_peers();
|
|
$masterRaw = vv_read_conf_raw('master.conf');
|
|
$restartLog = DB_DIR . '/container_restart_history.db';
|
|
|
|
// Config thresholds from master.conf
|
|
$cfg = [
|
|
'rw_soft_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_SOFT_GB') ?: 12),
|
|
'rw_medium_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_MEDIUM_GB') ?: 8),
|
|
'rw_hard_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_HARD_GB') ?: 6),
|
|
'rw_recover_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_RECOVER_GB') ?: 20),
|
|
'rw_load_soft' => (float)(vv_wd_scalar($masterRaw, 'RW_LOAD_SOFT_MULTIPLIER') ?: 2.0),
|
|
'rw_load_med' => (float)(vv_wd_scalar($masterRaw, 'RW_LOAD_MEDIUM_MULTIPLIER') ?: 3.0),
|
|
'sys_mem_gb' => (float)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_MEM_GB') ?: 4),
|
|
'sys_strikes' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_STRIKE_LIMIT') ?: 2),
|
|
'reboot_limit' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_REBOOT_LIMIT') ?: 3),
|
|
'reboot_window' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_REBOOT_WINDOW_HRS') ?: 12),
|
|
'restart_limit' => (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_CONTAINER_RESTART_LIMIT') ?: 3),
|
|
'startup_grace' => (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_STARTUP_GRACE') ?: 600),
|
|
'soft_mem_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_MEM_THRESHOLD') ?: 80),
|
|
'soft_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_CPU_THRESHOLD') ?: 80),
|
|
'hard_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'HARD_CPU_THRESHOLD') ?: 85),
|
|
'cpu_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'CPU_FAIL_LIMIT') ?: 2),
|
|
'resp_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'RESP_FAIL_LIMIT') ?: 2),
|
|
// Storage watchdog
|
|
'growth_gb' => (float)(vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_GROWTH_GB') ?: 2),
|
|
'log_max_gb' => (float)(vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_LOG_MAX_GB') ?: 2),
|
|
'stor_strike_lim'=> (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_STRIKE_LIMIT') ?: 3),
|
|
'truncate_logs' => vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_TRUNCATE_LOGS') === 'true',
|
|
// Network watchdog
|
|
'net_wd_enabled' => vv_wd_scalar($masterRaw, 'NETWORK_WATCHDOG_ENABLED') !== 'false',
|
|
'npm_strike_lim' => (int)(vv_wd_scalar($masterRaw, 'NETWORK_WATCHDOG_NPM_STRIKE_LIMIT') ?: 2),
|
|
'ts_check' => vv_wd_scalar($masterRaw, 'NETWORK_WATCHDOG_CHECK_TAILSCALE') !== 'false',
|
|
// Conf cache watchdog. It has no state of its own to report because it writes none until
|
|
// it runs, and it does not run: conf_cache_watchdog.sh:145-146 exits on either of these
|
|
// being off. Both gates ship so the page can say "off, and here is which switch", rather
|
|
// than omit a member of SYSTEM_WATCHDOG_SCRIPTS entirely and let its absence read as fine.
|
|
'fallback_on' => vv_wd_scalar($masterRaw, 'FALLBACK_ENABLED') === 'true',
|
|
'conf_sync_on' => vv_wd_scalar($masterRaw, 'CONF_SYNC_ENABLED') !== 'false',
|
|
// How often a cycle is supposed to happen, so "overdue" is measured against the schedule
|
|
// in force rather than a number written here. Read from the cron because that is where
|
|
// the answer lives — there is no conf key for it.
|
|
'cycle_min' => vv_wd_cycle_interval_min(),
|
|
];
|
|
|
|
// SSH key from current host conf
|
|
$myId = strtoupper($currentHost);
|
|
$myRaw = vv_read_conf_raw($currentHost . '.conf');
|
|
$mySshKey = vv_wd_scalar($myRaw, $myId . '_SSH_KEY');
|
|
|
|
// Known hosts
|
|
preg_match_all('/^\s*(HOST(\d+))(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $masterRaw, $m);
|
|
$hosts = [];
|
|
foreach ($m[1] as $i => $key) {
|
|
$hosts['host' . $m[2][$i]] = trim($m[3][$i]);
|
|
}
|
|
ksort($hosts);
|
|
if (!$hosts) $hosts = ['host1' => 'HOST1'];
|
|
|
|
$nodes = [];
|
|
foreach ($hosts as $slot => $hostname) {
|
|
$isMe = ($slot === $currentHost || $currentHost === 'unknown');
|
|
// Exact-key only, until now — the third copy of that lookup in this codebase and the
|
|
// third to render a live partner as dark. This mesh's conf name and tailnet name differ
|
|
// by one character, so HOST2 missed every time and its whole card read UNREACHABLE /
|
|
// "No data" while HOST2's own page showed the same watchdogs reporting OK.
|
|
$ts = vv_pt_peer_lookup($tsPeers, $hostname);
|
|
$ip = $ts['ip'] ?? null;
|
|
$raw = vv_read_conf_raw($slot . '.conf');
|
|
|
|
$remoteApiKey = vv_wd_scalar($raw, strtoupper($slot) . '_UNRAID_API_KEY');
|
|
|
|
if ($isMe) {
|
|
$system = vv_wd_local_system();
|
|
$states = vv_wd_local_states($restartLog);
|
|
} elseif ($ip && $mySshKey && $ts['online']) {
|
|
$remote = vv_wd_remote_data($ip, $mySshKey, $restartLog);
|
|
$system = $remote['system'] ?? null;
|
|
$states = $remote['states'] ?? null;
|
|
} elseif ($remoteApiKey) {
|
|
$remoteStats = vv_remote_hosts_stats();
|
|
$rs = $remoteStats[strtoupper($slot)] ?? null;
|
|
if ($rs && ($rs['available'] ?? false)) {
|
|
$totalGb = (float)($rs['mem_total_gb'] ?? 0);
|
|
$usedPct = (float)($rs['mem_used_pct'] ?? 0) / 100;
|
|
$totalB = (int)($totalGb * 1073741824);
|
|
$availB = (int)($totalB * (1 - $usedPct));
|
|
$system = [
|
|
'mem_total' => $totalB,
|
|
'mem_avail' => $availB,
|
|
'load1' => 0.0,
|
|
'cores' => (int)($rs['cpu_threads'] ?? 0),
|
|
'uptime' => (int)($rs['uptime_sec'] ?? 0),
|
|
'daemon_ok' => null,
|
|
'oom_count' => 0,
|
|
'api_only' => true,
|
|
];
|
|
} else {
|
|
$system = null;
|
|
}
|
|
$states = null;
|
|
} else {
|
|
$system = null;
|
|
$states = null;
|
|
}
|
|
|
|
$nodes[] = [
|
|
'slot' => $slot,
|
|
'id' => strtoupper($slot),
|
|
'hostname' => $hostname,
|
|
'is_me' => $isMe,
|
|
'ts_online' => $ts['online'],
|
|
'system' => $system,
|
|
'states' => $states,
|
|
'config' => vv_wd_node_config($slot, $raw, $masterRaw),
|
|
];
|
|
}
|
|
|
|
return [
|
|
'ts' => time(),
|
|
'cfg' => $cfg,
|
|
'nodes' => $nodes,
|
|
];
|
|
}
|