Gmer4Lfe
2cd1384786
Every existing exemption lasts until someone remembers to undo it, and nobody does — Healarr has sat in a pressure list since it was uninstalled and seven ignore entries name containers that are gone. A mute states when it ends and then ends, capped by WATCHDOG_MUTE_MAX_HOURS so temporary is enforced rather than intended. Applied where IGNORE_MAP is built, so all five check sites inherit it, and shown with its countdown because an invisible suppression is the thing being fixed.
599 lines
31 KiB
PHP
599 lines
31 KiB
PHP
<?php
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
// PURPOSE
|
|
// Watchdog page data layer. Collects what every watchdog has recorded — resource strikes,
|
|
// container restart history and the failed-container skip list, system strikes and OOM
|
|
// count, storage growth and log strikes, network/NPM strikes, and the reboot log — for
|
|
// this host and each partner.
|
|
//
|
|
// DESIGN PRINCIPLES
|
|
// Reads the strike counters; never resets them.
|
|
// Strike state belongs to the watchdog that owns it. A page that cleared strikes would
|
|
// silently undo an escalation the watchdog was deliberately building toward.
|
|
//
|
|
// One state file per watchdog, parsed independently.
|
|
// resource / container / system / storage / network each keep their own db. A watchdog
|
|
// that has never run leaves its file absent, which is a distinct and meaningful state.
|
|
//
|
|
// The skip list is surfaced, not hidden.
|
|
// docker_watchdog_failed.db records containers the docker watchdog has given up on.
|
|
// Those are exactly the ones an operator needs to see, so they are shown rather than
|
|
// filtered out of the healthy-looking list.
|
|
//
|
|
// OPERATIONAL SAFEGUARDS
|
|
// Every state read is suppressed and defaulted.
|
|
// @file_get_contents with a ?: fallback throughout — an absent or unreadable db yields
|
|
// empty/zero and that watchdog's card renders as "no data", never as a fatal.
|
|
//
|
|
// Absent counters read as zero, not as unknown-therefore-alarming.
|
|
// A watchdog that has not yet written state is reported quiet rather than as a
|
|
// problem, so a fresh boot does not light up the page with false strikes.
|
|
//
|
|
// Remote collection failures are per-node.
|
|
// One unreachable partner drops that node's card; the local host and every other
|
|
// partner still render.
|
|
//
|
|
// Read-only. Nothing here restarts a container, clears a strike, or triggers a reboot.
|
|
//
|
|
// EXPORTS
|
|
// Local vv_wd_local_system(), vv_wd_local_states()
|
|
// Remote vv_wd_remote_data(), vv_wd_node_config()
|
|
// Assembly vv_wd_all()
|
|
// Parsing vv_wd_bash_array(), vv_wd_bash_assoc(), vv_wd_scalar(), vv_wd_parse_kv(),
|
|
// vv_wd_parse_restart_log(), vv_wd_parse_skiplist(), vv_wd_parse_reboot_log(),
|
|
// vv_wd_parse_storage_state(), vv_wd_parse_network_state()
|
|
//
|
|
// CONFIGURATION
|
|
// STATE_DIR resource_watchdog_state.db, container_watchdog_state.db,
|
|
// docker_watchdog_failed.db, system_watchdog_state.db,
|
|
// system_watchdog_oom.db, storage/network watchdog state
|
|
// RW_CRITICAL_CONTAINERS containers the resource watchdog treats as critical
|
|
// HOST*_SSH_KEY used to collect partner watchdog state
|
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
require_once __DIR__ . '/common.php';
|
|
require_once __DIR__ . '/partnership.php';
|
|
|
|
// Watchdog tab data helpers
|
|
|
|
// Rows in system_watchdog_state.db that are bookkeeping, not strikes. The file is a flat key/value
|
|
// store shared by the counters and the watchdog's own liveness record, so the only thing separating
|
|
// the two is knowing which is which — there is no marker in the format.
|
|
//
|
|
// watchdog_cycle is a unix timestamp, written every cycle by stability_watchdog.sh:883 for
|
|
// docker_watchdog's stale-state guard. Published as a strike it reads as 1.7 billion of them.
|
|
//
|
|
// Add to this list, never to a shape test: a strike count and a timestamp are both positive
|
|
// integers, and a threshold like "bigger than a billion means it is a clock" is a rule that works
|
|
// until it doesn't and fails silently in the direction of hiding a real strike.
|
|
const VV_WD_SYS_BOOKKEEPING = ['watchdog_cycle'];
|
|
|
|
// Every container that exists on this host, running or not, for deciding whether a name in a
|
|
// watchdog list still refers to anything.
|
|
//
|
|
// `docker ps -a` rather than vv_docker_containers() + vv_docker_stopped(): those two are filtered
|
|
// by status and between them still miss states like restarting, and a container this page called
|
|
// "not installed" because it happened to be mid-restart would be worse than not checking at all.
|
|
// The whole point of the badge is that it is only ever shown when it is certainly true.
|
|
//
|
|
// Returns an empty array when docker cannot be reached, and callers must treat that as "unknown"
|
|
// rather than "nothing is installed" — otherwise a docker outage marks every entry dead.
|
|
function vv_wd_installed_containers(): array {
|
|
@exec("docker ps -a --format '{{.Names}}' 2>/dev/null", $out, $rc);
|
|
if ($rc !== 0) return [];
|
|
return array_values(array_filter(array_map('trim', $out), fn($n) => $n !== ''));
|
|
}
|
|
|
|
// Active timed mutes, as [name => ['left' => seconds, 'reason' => text]]. Expiry is a read-time
|
|
// comparison here for the same reason it is in wd_mute_active(): a mute ends when it says it does,
|
|
// whether or not anything has pruned the file since.
|
|
//
|
|
// Surfaced because a suppression nobody can see is the problem this feature exists to fix. An
|
|
// invisible exemption gets forgotten exactly like a permanent one — the only difference would be
|
|
// that this one also lies about how long it lasts.
|
|
function vv_wd_mutes(string $file = ''): array {
|
|
$path = $file ?: (vv_conf_vars()['WATCHDOG_MUTE_FILE'] ?? STATE_DIR . '/watchdog_mutes.db');
|
|
$raw = @file_get_contents($path);
|
|
if ($raw === false) return [];
|
|
$now = time();
|
|
$out = [];
|
|
foreach (explode("\n", $raw) as $line) {
|
|
$line = trim($line);
|
|
if ($line === '') continue;
|
|
$p = explode('|', $line);
|
|
if (count($p) < 2) continue;
|
|
$until = (int)$p[1];
|
|
if ($until <= $now || $p[0] === '') continue;
|
|
$out[$p[0]] = ['left' => $until - $now, 'reason' => trim($p[2] ?? '')];
|
|
}
|
|
return $out;
|
|
}
|
|
|
|
// How many minutes are supposed to pass between watchdog cycles.
|
|
//
|
|
// Parsed from varaverk.cron, which the scheduler regenerates at array start and is the only place
|
|
// the answer exists — the interval is a schedule, not a conf key. Reading it rather than assuming
|
|
// 15 means that changing the schedule also moves the point at which the page calls a cycle
|
|
// overdue, instead of leaving a constant here to drift out of agreement with reality.
|
|
//
|
|
// Falls back to 15 when the cron is missing or the entry is written in a form this does not
|
|
// recognise. A wrong-but-sane interval produces a slightly early or late warning; refusing to
|
|
// answer would remove the liveness check altogether, which is the failure being fixed.
|
|
function vv_wd_cycle_interval_min(): int {
|
|
$cron = @file_get_contents('/boot/config/plugins/varaverk/varaverk.cron') ?: '';
|
|
foreach (explode("\n", $cron) as $line) {
|
|
if (!str_contains($line, 'watchdog_orchestrator')) continue;
|
|
if (preg_match('#^\s*\*/(\d+)\s#', $line, $m)) return max(1, (int)$m[1]);
|
|
}
|
|
return 15;
|
|
}
|
|
|
|
// ── Conf array parser (bash arrays) ──────────────────────────────────────────
|
|
|
|
function vv_wd_bash_array(string $raw, string $varname): array {
|
|
return vv_parse_bash_array($raw, $varname);
|
|
}
|
|
|
|
function vv_wd_bash_assoc(string $raw, string $varname): array {
|
|
// declare -A VARNAME=( ["key"]=val ["key2"]=val2 )
|
|
if (!preg_match('/^\s*declare\s+-A\s+' . preg_quote($varname, '/') . '\s*=\s*\(\s*(.*?)\s*\)/ms', $raw, $m))
|
|
return [];
|
|
preg_match_all('/\["([^"]+)"\]\s*=\s*"?([^"\s\)]*)"?/', $m[1], $pairs);
|
|
$out = [];
|
|
foreach ($pairs[1] as $i => $k) $out[$k] = $pairs[2][$i];
|
|
return $out;
|
|
}
|
|
|
|
function vv_wd_scalar(string $raw, string $varname): string {
|
|
return vv_parse_conf_scalar($raw, $varname);
|
|
}
|
|
|
|
// ── State file parsers ────────────────────────────────────────────────────────
|
|
|
|
// Parses key:value format (with optional key=value mixed in)
|
|
function vv_wd_parse_kv(string $text): array {
|
|
$out = [];
|
|
foreach (explode("\n", $text) as $line) {
|
|
$line = trim($line);
|
|
if (!$line) continue;
|
|
if (str_contains($line, ':')) {
|
|
[$k, $v] = explode(':', $line, 2);
|
|
$out[trim($k)] = trim($v);
|
|
} elseif (str_contains($line, '=')) {
|
|
[$k, $v] = explode('=', $line, 2);
|
|
$out[trim($k)] = trim($v, '"\'');
|
|
}
|
|
}
|
|
return $out;
|
|
}
|
|
|
|
// Restart log: "container|2026-08-02 13:15:07" one per line.
|
|
//
|
|
// The second field is a formatted local timestamp, not an epoch — docker_watchdog.sh writes it
|
|
// with date '+%Y-%m-%d %H:%M:%S' because its own rolling-window trim compares the strings
|
|
// lexically in awk, which is correct for that format. This function used to cast it with (int),
|
|
// which stops at the first non-digit and yielded 2026 for every line ever written. 2026 is below
|
|
// any plausible cutoff, so every entry was discarded and both restart panels — the Monitor card
|
|
// and the Watchdog page — were permanently empty. Not visibly broken: an empty list reads
|
|
// exactly like "nothing has restarted", which is the answer you least want to be wrong about
|
|
// during a restart loop.
|
|
//
|
|
// Parsed, not reformatted. Changing what the watchdog writes would strand every existing entry
|
|
// and every awk comparison in Tools/watchdog_skip_list_manager.sh.
|
|
function vv_wd_parse_restart_log(string $text, int $windowSeconds = 86400): array {
|
|
$cutoff = time() - $windowSeconds;
|
|
$entries = [];
|
|
foreach (explode("\n", trim($text)) as $line) {
|
|
$line = trim($line);
|
|
if (!$line || !str_contains($line, '|')) continue;
|
|
[$name, $raw] = explode('|', $line, 2);
|
|
$ts = vv_wd_restart_ts(trim($raw));
|
|
if ($ts >= $cutoff) $entries[] = ['name' => trim($name), 'ts' => $ts];
|
|
}
|
|
usort($entries, fn($a, $b) => $b['ts'] - $a['ts']);
|
|
return $entries;
|
|
}
|
|
|
|
// vv_wd_restart_ts() lives in common.php — include/monitor.php parses the same file and does not
|
|
// include this one, so a helper defined here would be a fatal on the dashboard.
|
|
|
|
// Skip list: one container name per line
|
|
function vv_wd_parse_skiplist(string $text): array {
|
|
return array_values(array_filter(array_map('trim', explode("\n", $text))));
|
|
}
|
|
|
|
// Reboot log: one timestamp per line
|
|
function vv_wd_parse_reboot_log(string $text, int $windowHrs = 12): array {
|
|
$cutoff = time() - ($windowHrs * 3600);
|
|
$entries = [];
|
|
foreach (explode("\n", trim($text)) as $line) {
|
|
$ts = (int)trim($line);
|
|
if ($ts > 0 && $ts >= $cutoff) $entries[] = $ts;
|
|
}
|
|
rsort($entries);
|
|
return $entries;
|
|
}
|
|
|
|
// ── Local system snapshot ─────────────────────────────────────────────────────
|
|
|
|
function vv_wd_local_system(): array {
|
|
$sys = vv_system_info();
|
|
$mem = vv_memory_breakdown();
|
|
$loadRaw = @file_get_contents('/proc/loadavg') ?: '0';
|
|
$daemonOk = (trim(shell_exec('docker info >/dev/null 2>&1; echo $?') ?: '1') === '0');
|
|
$oomCount = (int)trim(@file_get_contents(STATE_DIR . '/system_watchdog_oom.db') ?: '0');
|
|
$cores = (int)($sys['cpu_cores'] ?: (int)(trim(shell_exec('nproc 2>/dev/null') ?: '1')));
|
|
return [
|
|
'mem_total' => (int)($mem['total_kb'] * 1024),
|
|
'mem_avail' => (int)($mem['free_kb'] * 1024),
|
|
'load1' => (float)explode(' ', trim($loadRaw))[0],
|
|
'cores' => $cores,
|
|
'uptime' => $sys['uptime_sec'] ?? 0,
|
|
'daemon_ok' => $daemonOk,
|
|
'oom_count' => $oomCount,
|
|
];
|
|
}
|
|
|
|
// ── Storage / network state parsers ──────────────────────────────────────────
|
|
|
|
function vv_wd_parse_storage_state(string $raw): array {
|
|
$growth = []; $log = [];
|
|
foreach (explode("\n", $raw) as $line) {
|
|
$line = trim($line);
|
|
if (!$line || !str_contains($line, ':')) continue;
|
|
[$k, $v] = explode(':', $line, 2);
|
|
$count = (int)trim($v);
|
|
if ($count <= 0) continue;
|
|
$key = trim($k);
|
|
if (str_starts_with($key, 'appdata_growth_'))
|
|
$growth[substr($key, strlen('appdata_growth_'))] = $count;
|
|
elseif (str_starts_with($key, 'appdata_log_'))
|
|
$log[substr($key, strlen('appdata_log_'))] = $count;
|
|
}
|
|
return ['growth_strikes' => $growth, 'log_strikes' => $log];
|
|
}
|
|
|
|
function vv_wd_parse_network_state(string $raw): array {
|
|
$npm = 0;
|
|
foreach (explode("\n", $raw) as $line) {
|
|
$line = trim($line);
|
|
if (str_starts_with($line, 'npm:')) $npm = (int)trim(substr($line, 4));
|
|
}
|
|
return ['npm_strikes' => $npm];
|
|
}
|
|
|
|
// ── Local state files ─────────────────────────────────────────────────────────
|
|
|
|
function vv_wd_local_states(string $restartLogPath): array {
|
|
$rwRaw = @file_get_contents(STATE_DIR . '/resource_watchdog_state.db') ?: '';
|
|
$dockRaw = @file_get_contents(STATE_DIR . '/container_watchdog_state.db') ?: '';
|
|
$skipRaw = @file_get_contents(STATE_DIR . '/docker_watchdog_failed.db') ?: '';
|
|
$sysRaw = @file_get_contents(STATE_DIR . '/system_watchdog_state.db') ?: '';
|
|
$rebootRaw = @file_get_contents(STATE_DIR . '/system_watchdog_reboots.db') ?: '';
|
|
$restartRaw= @file_get_contents($restartLogPath) ?: '';
|
|
$storRaw = @file_get_contents(STATE_DIR . '/storage_watchdog_state.db') ?: '';
|
|
$netWdRaw = @file_get_contents(STATE_DIR . '/network_watchdog_state.db') ?: '';
|
|
|
|
$rw = vv_wd_parse_kv($rwRaw);
|
|
$dock = vv_wd_parse_kv($dockRaw);
|
|
$sys = vv_wd_parse_kv($sysRaw);
|
|
|
|
// Container strikes: everything in docker state that isn't a flag
|
|
$strikes = [];
|
|
foreach ($dock as $k => $v) {
|
|
if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0)
|
|
$strikes[$k] = (int)$v;
|
|
}
|
|
|
|
// Stability strikes: everything in sys state that is actually a strike.
|
|
//
|
|
// The old guard was `!str_contains($k, '=')`, written on the belief that the bookkeeping rows
|
|
// in system_watchdog_state.db keep their `=` inside the key because the file mixes separators
|
|
// — `ram:0` and `sshd:0` beside `kernel_oops_count=0` and `watchdog_cycle=...`. They do not:
|
|
// vv_wd_parse_kv() splits on both, so the guard never matched anything and every bookkeeping
|
|
// row was published as a strike. watchdog_cycle is a unix timestamp written once per cycle by
|
|
// stability_watchdog.sh:883 as a liveness heartbeat, so the Stability card was drawing
|
|
// "watchdog cycle — 1786716931" next to a limit of 2.
|
|
$sysStrikes = [];
|
|
foreach ($sys as $k => $v) {
|
|
if (in_array($k, VV_WD_SYS_BOOKKEEPING, true)) continue;
|
|
if ((int)$v > 0) $sysStrikes[$k] = (int)$v;
|
|
}
|
|
|
|
// Growth baseline info (container count + age in seconds)
|
|
$growthFile = STATE_DIR . '/watchdog_appdata_growth.db';
|
|
$baselineCount = file_exists($growthFile) ? max(0, count(file($growthFile)) - 0) : 0;
|
|
$baselineAgeSec = file_exists($growthFile) ? time() - (int)filemtime($growthFile) : null;
|
|
|
|
return [
|
|
'rw_level' => (int)($rw['rm_action_level'] ?? 0),
|
|
'rw_recover' => (int)($rw['rm_recover_cycles'] ?? 0),
|
|
'rw_paused' => array_filter(explode(',', $rw['rm_paused_containers'] ?? '')),
|
|
'rw_stopped' => array_filter(explode(',', $rw['rm_stopped_containers'] ?? '')),
|
|
'mem_shutdown' => ($rw['mem_shutdown_active'] ?? 'false') === 'true',
|
|
'daemon_strikes' => (int)($dock['daemon_strikes'] ?? 0),
|
|
'daemon_restart' => ($dock['daemon_restarted_flag'] ?? 'false') === 'true',
|
|
'ctr_strikes' => $strikes,
|
|
'skiplist' => vv_wd_parse_skiplist($skipRaw),
|
|
'sys_strikes' => $sysStrikes,
|
|
// The heartbeat, not a strike — see VV_WD_SYS_BOOKKEEPING. stability_watchdog.sh writes it
|
|
// last in the chain, so a fresh value means a whole cycle completed rather than started.
|
|
'last_cycle' => (int)($sys['watchdog_cycle'] ?? 0),
|
|
'mutes' => vv_wd_mutes(),
|
|
'reboots' => vv_wd_parse_reboot_log($rebootRaw),
|
|
'restarts' => vv_wd_parse_restart_log($restartRaw),
|
|
'storage_wd' => vv_wd_parse_storage_state($storRaw) + [
|
|
'baseline_count' => $baselineCount,
|
|
'baseline_age_sec' => $baselineAgeSec,
|
|
],
|
|
'network_wd' => vv_wd_parse_network_state($netWdRaw),
|
|
];
|
|
}
|
|
|
|
// ── Remote data via SSH ───────────────────────────────────────────────────────
|
|
|
|
function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath): array {
|
|
// Bundle into one SSH call.
|
|
// /proc/meminfo is passed as a raw section (not awk-parsed) to avoid quoting
|
|
// fragility — escapeshellarg() single-quotes the whole command so awk \$2
|
|
// inside double-quotes is unreliable across Unraid builds.
|
|
// State files live under the REMOTE's own SCRIPTS_DIR (may differ from ours in flash mode),
|
|
// so it is resolved on the far side — same idiom as vv_remote_state_cmd().
|
|
//
|
|
// The layout is probed rather than assumed. State moved from SCRIPTS_DIR/State_Files to
|
|
// DATA_DIR/state, and the histories from DATA_DIR's root into DATA_DIR/db, but a partner is
|
|
// not guaranteed to have pulled that yet — and this is the call that reports whether the
|
|
// partner's watchdogs are healthy. Guessing wrong returns empty strings for every state
|
|
// file, which reads as "partner has no strikes" rather than as an error. Probing costs one
|
|
// directory test and makes the answer correct in both directions, which also means the two
|
|
// hosts can be upgraded in either order.
|
|
$restartLogName = basename($restartLogPath);
|
|
$cmd = 'sd=$(grep -m1 SCRIPTS_DIR= /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null'
|
|
. ' | cut -d\'"\' -f2); sd="${sd:-/boot/config/plugins/varaverk}"; '
|
|
. 'sf="$sd/data/state"; [ -d "$sf" ] || sf="$sd/State_Files"; '
|
|
. 'db="$sd/data/db"; [ -d "$db" ] || db="$sd/data"; '
|
|
. "printf 'UPTIME:%s\nLOAD:%s\nCORES:%s\nDAEMON:%s\nOOM:%s\nBASELINECOUNT:%s\nBASELINEAGE:%s\n---MEMINFO---\n%s\n---RW---\n%s\n---DOCK---\n%s\n---SKIP---\n%s\n---SYS---\n%s\n---REBOOT---\n%s\n---RESTART---\n%s\n---STORAGE---\n%s\n---NETWORK---\n%s\n' "
|
|
. '"$(cat /proc/uptime|cut -d\" \" -f1)" '
|
|
. '"$(cat /proc/loadavg|cut -d\" \" -f1)" '
|
|
. '"$(nproc)" '
|
|
. '"$(docker info >/dev/null 2>&1 && echo ok || echo err)" '
|
|
. '"$(cat "$sf/system_watchdog_oom.db" 2>/dev/null||echo 0)" '
|
|
. '"$(wc -l < "$sf/watchdog_appdata_growth.db" 2>/dev/null||echo 0)" '
|
|
. '"$(stat -c %Y "$sf/watchdog_appdata_growth.db" 2>/dev/null||echo 0)" '
|
|
. '"$(cat /proc/meminfo 2>/dev/null)" '
|
|
. '"$(cat "$sf/resource_watchdog_state.db" 2>/dev/null)" '
|
|
. '"$(cat "$sf/container_watchdog_state.db" 2>/dev/null)" '
|
|
. '"$(cat "$sf/docker_watchdog_failed.db" 2>/dev/null)" '
|
|
. '"$(cat "$sf/system_watchdog_state.db" 2>/dev/null)" '
|
|
. '"$(cat "$sf/system_watchdog_reboots.db" 2>/dev/null)" '
|
|
. '"$(cat "$db/' . $restartLogName . '" 2>/dev/null)" '
|
|
. '"$(cat "$sf/storage_watchdog_state.db" 2>/dev/null)" '
|
|
. '"$(cat "$sf/network_watchdog_state.db" 2>/dev/null)"';
|
|
|
|
$out = vv_pt_ssh($ip, $sshKey, $cmd, 8);
|
|
if (!$out) return null;
|
|
|
|
// Parse sections
|
|
$sections = preg_split('/^---\w+---$/m', $out);
|
|
$header = $sections[0] ?? '';
|
|
$memInfoRaw = $sections[1] ?? '';
|
|
$rwRaw = $sections[2] ?? '';
|
|
$dockRaw = $sections[3] ?? '';
|
|
$skipRaw = $sections[4] ?? '';
|
|
$sysRaw = $sections[5] ?? '';
|
|
$rebootRaw = $sections[6] ?? '';
|
|
$restartRaw = $sections[7] ?? '';
|
|
$storRaw = $sections[8] ?? '';
|
|
$netWdRaw = $sections[9] ?? '';
|
|
|
|
// Parse header lines
|
|
$hdr = [];
|
|
foreach (explode("\n", $header) as $line) {
|
|
if (preg_match('/^(\w+):(.*)$/', trim($line), $m)) $hdr[$m[1]] = trim($m[2]);
|
|
}
|
|
|
|
// Parse /proc/meminfo section — no awk, no quoting issues
|
|
$memTotal = 0; $memAvail = 0;
|
|
if (preg_match('/^MemTotal:\s+(\d+)/m', $memInfoRaw, $m)) $memTotal = (int)$m[1] * 1024;
|
|
if (preg_match('/^MemAvailable:\s+(\d+)/m', $memInfoRaw, $m)) $memAvail = (int)$m[1] * 1024;
|
|
|
|
$rw = vv_wd_parse_kv($rwRaw);
|
|
$dock = vv_wd_parse_kv($dockRaw);
|
|
$sys = vv_wd_parse_kv($sysRaw);
|
|
|
|
$strikes = [];
|
|
foreach ($dock as $k => $v) {
|
|
if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0)
|
|
$strikes[$k] = (int)$v;
|
|
}
|
|
// Same exclusion as the local path above — a partner's heartbeat is no more a strike than
|
|
// this host's. The two builders are maintained together.
|
|
$sysStrikes = [];
|
|
foreach ($sys as $k => $v) {
|
|
if (in_array($k, VV_WD_SYS_BOOKKEEPING, true)) continue;
|
|
if ((int)$v > 0) $sysStrikes[$k] = (int)$v;
|
|
}
|
|
|
|
$baselineCount = (int)($hdr['BASELINECOUNT'] ?? 0);
|
|
$baselineTs = (int)($hdr['BASELINEAGE'] ?? 0);
|
|
$baselineAgeSec = $baselineTs > 0 ? time() - $baselineTs : null;
|
|
|
|
return [
|
|
'system' => [
|
|
'mem_total' => $memTotal,
|
|
'mem_avail' => $memAvail,
|
|
'load1' => (float)($hdr['LOAD'] ?? 0),
|
|
'cores' => (int)($hdr['CORES'] ?? 1),
|
|
'uptime' => (int)($hdr['UPTIME'] ?? 0),
|
|
'daemon_ok' => ($hdr['DAEMON'] ?? '') === 'ok',
|
|
'oom_count' => (int)($hdr['OOM'] ?? 0),
|
|
],
|
|
'states' => [
|
|
'rw_level' => (int)($rw['rm_action_level'] ?? 0),
|
|
'rw_recover' => (int)($rw['rm_recover_cycles'] ?? 0),
|
|
'rw_paused' => array_filter(explode(',', $rw['rm_paused_containers'] ?? '')),
|
|
'rw_stopped' => array_filter(explode(',', $rw['rm_stopped_containers'] ?? '')),
|
|
'mem_shutdown' => ($rw['mem_shutdown_active'] ?? 'false') === 'true',
|
|
'daemon_strikes'=> (int)($dock['daemon_strikes'] ?? 0),
|
|
'daemon_restart'=> ($dock['daemon_restarted_flag'] ?? 'false') === 'true',
|
|
'ctr_strikes' => $strikes,
|
|
'skiplist' => vv_wd_parse_skiplist($skipRaw),
|
|
'sys_strikes' => $sysStrikes,
|
|
'last_cycle' => (int)($sys['watchdog_cycle'] ?? 0),
|
|
'reboots' => vv_wd_parse_reboot_log($rebootRaw),
|
|
'restarts' => vv_wd_parse_restart_log($restartRaw),
|
|
'storage_wd' => vv_wd_parse_storage_state($storRaw) + [
|
|
'baseline_count' => $baselineCount,
|
|
'baseline_age_sec' => $baselineAgeSec,
|
|
],
|
|
'network_wd' => vv_wd_parse_network_state($netWdRaw),
|
|
],
|
|
];
|
|
}
|
|
|
|
// ── Config inventory ──────────────────────────────────────────────────────────
|
|
|
|
function vv_wd_node_config(string $slot, string $raw, string $masterRaw): array {
|
|
$id = strtoupper($slot);
|
|
return [
|
|
'monitored' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINERS"),
|
|
'urls' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINER_URLS"),
|
|
'required' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_REQUIRED_CONTAINERS"),
|
|
'ignore' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_SCAN_IGNORE"),
|
|
'pause_list' => vv_wd_bash_array($raw, "{$id}_RW_PAUSE_CONTAINERS"),
|
|
'stop_list' => vv_wd_bash_array($raw, "{$id}_RW_STOP_CONTAINERS"),
|
|
'critical' => vv_wd_bash_array($masterRaw, "RW_CRITICAL_CONTAINERS"),
|
|
// Network watchdog — host-specific
|
|
'ddns_domain' => vv_wd_scalar($raw, "{$id}_NETWORK_WATCHDOG_DDNS_DOMAIN"),
|
|
'ddns_container' => vv_wd_scalar($raw, "{$id}_NETWORK_WATCHDOG_DDNS_CONTAINER"),
|
|
'npm_url' => vv_wd_scalar($raw, "{$id}_NETWORK_WATCHDOG_NPM_URL"),
|
|
// Storage watchdog — host-specific appdata suppress ceilings
|
|
'appdata_sizes' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_APPDATA_SIZES"),
|
|
];
|
|
}
|
|
|
|
// ── Entry point ───────────────────────────────────────────────────────────────
|
|
|
|
function vv_wd_all(): array {
|
|
$currentHost = vv_detect_host();
|
|
$tsPeers = vv_pt_ts_peers();
|
|
$masterRaw = vv_read_conf_raw('master.conf');
|
|
$restartLog = DB_DIR . '/container_restart_history.db';
|
|
|
|
// Config thresholds from master.conf
|
|
$cfg = [
|
|
'rw_soft_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_SOFT_GB') ?: 12),
|
|
'rw_medium_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_MEDIUM_GB') ?: 8),
|
|
'rw_hard_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_HARD_GB') ?: 6),
|
|
'rw_recover_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_RECOVER_GB') ?: 20),
|
|
'rw_load_soft' => (float)(vv_wd_scalar($masterRaw, 'RW_LOAD_SOFT_MULTIPLIER') ?: 2.0),
|
|
'rw_load_med' => (float)(vv_wd_scalar($masterRaw, 'RW_LOAD_MEDIUM_MULTIPLIER') ?: 3.0),
|
|
'sys_mem_gb' => (float)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_MEM_GB') ?: 4),
|
|
'sys_strikes' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_STRIKE_LIMIT') ?: 2),
|
|
'reboot_limit' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_REBOOT_LIMIT') ?: 3),
|
|
'reboot_window' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_REBOOT_WINDOW_HRS') ?: 12),
|
|
'restart_limit' => (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_CONTAINER_RESTART_LIMIT') ?: 3),
|
|
'startup_grace' => (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_STARTUP_GRACE') ?: 600),
|
|
'soft_mem_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_MEM_THRESHOLD') ?: 80),
|
|
'soft_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_CPU_THRESHOLD') ?: 80),
|
|
'hard_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'HARD_CPU_THRESHOLD') ?: 85),
|
|
'cpu_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'CPU_FAIL_LIMIT') ?: 2),
|
|
'resp_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'RESP_FAIL_LIMIT') ?: 2),
|
|
// Storage watchdog
|
|
'growth_gb' => (float)(vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_GROWTH_GB') ?: 2),
|
|
'log_max_gb' => (float)(vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_LOG_MAX_GB') ?: 2),
|
|
'stor_strike_lim'=> (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_STRIKE_LIMIT') ?: 3),
|
|
'truncate_logs' => vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_TRUNCATE_LOGS') === 'true',
|
|
// Network watchdog
|
|
'net_wd_enabled' => vv_wd_scalar($masterRaw, 'NETWORK_WATCHDOG_ENABLED') !== 'false',
|
|
'npm_strike_lim' => (int)(vv_wd_scalar($masterRaw, 'NETWORK_WATCHDOG_NPM_STRIKE_LIMIT') ?: 2),
|
|
'ts_check' => vv_wd_scalar($masterRaw, 'NETWORK_WATCHDOG_CHECK_TAILSCALE') !== 'false',
|
|
// Conf cache watchdog. It has no state of its own to report because it writes none until
|
|
// it runs, and it does not run: conf_cache_watchdog.sh:145-146 exits on either of these
|
|
// being off. Both gates ship so the page can say "off, and here is which switch", rather
|
|
// than omit a member of SYSTEM_WATCHDOG_SCRIPTS entirely and let its absence read as fine.
|
|
'fallback_on' => vv_wd_scalar($masterRaw, 'FALLBACK_ENABLED') === 'true',
|
|
'conf_sync_on' => vv_wd_scalar($masterRaw, 'CONF_SYNC_ENABLED') !== 'false',
|
|
// How often a cycle is supposed to happen, so "overdue" is measured against the schedule
|
|
// in force rather than a number written here. Read from the cron because that is where
|
|
// the answer lives — there is no conf key for it.
|
|
'cycle_min' => vv_wd_cycle_interval_min(),
|
|
];
|
|
|
|
// SSH key from current host conf
|
|
$myId = strtoupper($currentHost);
|
|
$myRaw = vv_read_conf_raw($currentHost . '.conf');
|
|
$mySshKey = vv_wd_scalar($myRaw, $myId . '_SSH_KEY');
|
|
|
|
// Known hosts
|
|
preg_match_all('/^\s*(HOST(\d+))(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $masterRaw, $m);
|
|
$hosts = [];
|
|
foreach ($m[1] as $i => $key) {
|
|
$hosts['host' . $m[2][$i]] = trim($m[3][$i]);
|
|
}
|
|
ksort($hosts);
|
|
if (!$hosts) $hosts = ['host1' => 'HOST1'];
|
|
|
|
$nodes = [];
|
|
foreach ($hosts as $slot => $hostname) {
|
|
$isMe = ($slot === $currentHost || $currentHost === 'unknown');
|
|
$tsLabel = strtolower($hostname);
|
|
$ts = $tsPeers[$tsLabel] ?? ['online' => null, 'active' => false, 'ip' => null];
|
|
$ip = $ts['ip'] ?? null;
|
|
$raw = vv_read_conf_raw($slot . '.conf');
|
|
|
|
$remoteApiKey = vv_wd_scalar($raw, strtoupper($slot) . '_UNRAID_API_KEY');
|
|
|
|
if ($isMe) {
|
|
$system = vv_wd_local_system();
|
|
$states = vv_wd_local_states($restartLog);
|
|
} elseif ($ip && $mySshKey && $ts['online']) {
|
|
$remote = vv_wd_remote_data($ip, $mySshKey, $restartLog);
|
|
$system = $remote['system'] ?? null;
|
|
$states = $remote['states'] ?? null;
|
|
} elseif ($remoteApiKey) {
|
|
$remoteStats = vv_remote_hosts_stats();
|
|
$rs = $remoteStats[strtoupper($slot)] ?? null;
|
|
if ($rs && ($rs['available'] ?? false)) {
|
|
$totalGb = (float)($rs['mem_total_gb'] ?? 0);
|
|
$usedPct = (float)($rs['mem_used_pct'] ?? 0) / 100;
|
|
$totalB = (int)($totalGb * 1073741824);
|
|
$availB = (int)($totalB * (1 - $usedPct));
|
|
$system = [
|
|
'mem_total' => $totalB,
|
|
'mem_avail' => $availB,
|
|
'load1' => 0.0,
|
|
'cores' => (int)($rs['cpu_threads'] ?? 0),
|
|
'uptime' => (int)($rs['uptime_sec'] ?? 0),
|
|
'daemon_ok' => null,
|
|
'oom_count' => 0,
|
|
'api_only' => true,
|
|
];
|
|
} else {
|
|
$system = null;
|
|
}
|
|
$states = null;
|
|
} else {
|
|
$system = null;
|
|
$states = null;
|
|
}
|
|
|
|
$nodes[] = [
|
|
'slot' => $slot,
|
|
'id' => strtoupper($slot),
|
|
'hostname' => $hostname,
|
|
'is_me' => $isMe,
|
|
'ts_online' => $ts['online'],
|
|
'system' => $system,
|
|
'states' => $states,
|
|
'config' => vv_wd_node_config($slot, $raw, $masterRaw),
|
|
];
|
|
}
|
|
|
|
return [
|
|
'ts' => time(),
|
|
'cfg' => $cfg,
|
|
'nodes' => $nodes,
|
|
];
|
|
}
|