FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Plugin: common.php library, watchdog expansion, monitor + scheduler improvements

Browse Source 
PHP architecture:
- Extract common.php from monitor.php — shared system functions (vv_system_info,
  vv_memory_breakdown, vv_remote_hosts_stats, disk/GPU/UPS/network/docker/parity, etc.)
  now live in one place; monitor.php and watchdog.php both require common.php
- Add unraid_api.php as explicit include (was implicit via config.php chain)
- confform.php: add missing require_once config.php (implicit dep made explicit)
- Delete orphaned pages/docs.php and pages/config.php (absorbed into scheduler)

Watchdog page:
- Add Storage watchdog card (growth + log strikes, baseline age, suppress ceilings)
- Add Network watchdog card (NPM strikes, DDNS domain/container, NPM URL)
- One host per row layout — all 5 watchdog cards equally spaced via inner grid
- Watchdog now uses Unraid API for local system stats; remote nodes with API key
  but no SSH get system info from vv_remote_hosts_stats() with api_only flag
- SSH bundle: /proc/meminfo passed as raw section instead of awk-parsed header
  fields — fixes RAM showing 0 on remote hosts where awk quoting was unreliable
- vv_wd_local_system() rewritten as thin wrapper over vv_system_info() + vv_memory_breakdown()

Monitor page:
- Watchdog card: add stability strikes, storage watchdog strikes, network NPM status,
  live system stats (rootfs/log/tmp %, RAM free, load, CPU temp, zombies, NIC, sshd)
- Row height: switch from max-height on cards to minmax(0, calc(...)) on grid track —
  all cards in a row now fill to the tallest card's height correctly (fixes Pools card
  being shorter than neighbours)

Scheduler page:
- Add Tools section above Custom Scripts — lists Tools/*.sh with run/dry-run/cron/log
- vv_tools_scripts() function in scheduler.php include

Tools:
- Add docker_prune_images.sh — removes dangling Docker images; --dry-run and --status modes
This commit is contained in:
Gmer4Lfe
2026-05-29 23:33:52 -04:00
parent 8f05f0d27c
commit 86048b32d3
13 changed files with 2072 additions and 898 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Plugin/unraid/include/watchdog.php
+135 -53
View File
@@ -1,8 +1,8 @@
<?php
// Watchdog tab data helpers
require_once __DIR__ . '/common.php';
require_once __DIR__ . '/partnership.php';
require_once __DIR__ . '/config.php';
require_once __DIR__ . '/partnership.php'; // vv_pt_ssh(), vv_pt_ts_peers()
// Watchdog tab data helpers
// ── Conf array parser (bash arrays) ──────────────────────────────────────────
@@ -83,40 +83,51 @@ function vv_wd_parse_reboot_log(string $text, int $windowHrs = 12): array {
// ── Local system snapshot ─────────────────────────────────────────────────────
function vv_wd_local_system(): array {
// RAM
$memRaw = file_exists('/proc/meminfo') ? file_get_contents('/proc/meminfo') : '';
$memTotal = 0; $memAvail = 0;
if (preg_match('/^MemTotal:\s+(\d+)/m', $memRaw, $m)) $memTotal = (int)$m[1] * 1024;
if (preg_match('/^MemAvailable:\s+(\d+)/m', $memRaw, $m)) $memAvail = (int)$m[1] * 1024;
// Load + cores
$loadRaw = file_exists('/proc/loadavg') ? file_get_contents('/proc/loadavg') : '0 0 0';
$loadParts = explode(' ', trim($loadRaw));
$load1 = (float)($loadParts[0] ?? 0);
$cores = (int)(trim(shell_exec('nproc 2>/dev/null') ?: '1'));
// Uptime
$uptimeRaw = file_exists('/proc/uptime') ? file_get_contents('/proc/uptime') : '0';
$uptime = (int)explode(' ', $uptimeRaw)[0];
// Docker daemon alive
$sys = vv_system_info();
$mem = vv_memory_breakdown();
$loadRaw = @file_get_contents('/proc/loadavg') ?: '0';
$daemonOk = (trim(shell_exec('docker info >/dev/null 2>&1; echo $?') ?: '1') === '0');
// OOM count (from stability watchdog OOM file — just the prev cycle count)
$oomFile = '/tmp/system_watchdog_oom.db';
$oomCount = file_exists($oomFile) ? (int)trim(file_get_contents($oomFile)) : 0;
$oomCount = (int)trim(@file_get_contents('/tmp/system_watchdog_oom.db') ?: '0');
$cores = (int)($sys['cpu_cores'] ?: (int)(trim(shell_exec('nproc 2>/dev/null') ?: '1')));
return [
'mem_total' => $memTotal,
'mem_avail' => $memAvail,
'load1' => $load1,
'mem_total' => (int)($mem['total_kb'] * 1024),
'mem_avail' => (int)($mem['free_kb'] * 1024),
'load1' => (float)explode(' ', trim($loadRaw))[0],
'cores' => $cores,
'uptime' => $uptime,
'uptime' => $sys['uptime_sec'] ?? 0,
'daemon_ok' => $daemonOk,
'oom_count' => $oomCount,
];
}
// ── Storage / network state parsers ──────────────────────────────────────────
function vv_wd_parse_storage_state(string $raw): array {
$growth = []; $log = [];
foreach (explode("\n", $raw) as $line) {
$line = trim($line);
if (!$line || !str_contains($line, ':')) continue;
[$k, $v] = explode(':', $line, 2);
$count = (int)trim($v);
if ($count <= 0) continue;
$key = trim($k);
if (str_starts_with($key, 'appdata_growth_'))
$growth[substr($key, strlen('appdata_growth_'))] = $count;
elseif (str_starts_with($key, 'appdata_log_'))
$log[substr($key, strlen('appdata_log_'))] = $count;
}
return ['growth_strikes' => $growth, 'log_strikes' => $log];
}
function vv_wd_parse_network_state(string $raw): array {
$npm = 0;
foreach (explode("\n", $raw) as $line) {
$line = trim($line);
if (str_starts_with($line, 'npm:')) $npm = (int)trim(substr($line, 4));
}
return ['npm_strikes' => $npm];
}
// ── Local state files ─────────────────────────────────────────────────────────
function vv_wd_local_states(string $restartLogPath): array {
@@ -126,6 +137,8 @@ function vv_wd_local_states(string $restartLogPath): array {
$sysRaw = @file_get_contents('/tmp/system_watchdog_state.db') ?: '';
$rebootRaw = @file_get_contents('/boot/config/system_watchdog_reboots.db')?: '';
$restartRaw= @file_get_contents($restartLogPath) ?: '';
$storRaw = @file_get_contents('/tmp/storage_watchdog_state.db') ?: '';
$netWdRaw = @file_get_contents('/tmp/network_watchdog_state.db') ?: '';
$rw = vv_wd_parse_kv($rwRaw);
$dock = vv_wd_parse_kv($dockRaw);
@@ -145,6 +158,11 @@ function vv_wd_local_states(string $restartLogPath): array {
$sysStrikes[$k] = (int)$v;
}
// Growth baseline info (container count + age in seconds)
$growthFile = '/tmp/watchdog_appdata_growth.db';
$baselineCount = file_exists($growthFile) ? max(0, count(file($growthFile)) - 0) : 0;
$baselineAgeSec = file_exists($growthFile) ? time() - (int)filemtime($growthFile) : null;
return [
'rw_level' => (int)($rw['rm_action_level'] ?? 0),
'rw_recover' => (int)($rw['rm_recover_cycles'] ?? 0),
@@ -158,40 +176,54 @@ function vv_wd_local_states(string $restartLogPath): array {
'sys_strikes' => $sysStrikes,
'reboots' => vv_wd_parse_reboot_log($rebootRaw),
'restarts' => vv_wd_parse_restart_log($restartRaw),
'storage_wd' => vv_wd_parse_storage_state($storRaw) + [
'baseline_count' => $baselineCount,
'baseline_age_sec' => $baselineAgeSec,
],
'network_wd' => vv_wd_parse_network_state($netWdRaw),
];
}
// ── Remote data via SSH ───────────────────────────────────────────────────────
function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath): array {
// Bundle into one SSH call
$cmd = "printf 'UPTIME:%s\nLOAD:%s\nCORES:%s\nMEMTOTAL:%s\nMEMAVAIL:%s\nDAEMON:%s\nOOM:%s\n---RW---\n%s\n---DOCK---\n%s\n---SKIP---\n%s\n---SYS---\n%s\n---REBOOT---\n%s\n---RESTART---\n%s\n' "
// Bundle into one SSH call.
// /proc/meminfo is passed as a raw section (not awk-parsed) to avoid quoting
// fragility — escapeshellarg() single-quotes the whole command so awk \$2
// inside double-quotes is unreliable across Unraid builds.
$cmd = "printf 'UPTIME:%s\nLOAD:%s\nCORES:%s\nDAEMON:%s\nOOM:%s\nBASELINECOUNT:%s\nBASELINEAGE:%s\n---MEMINFO---\n%s\n---RW---\n%s\n---DOCK---\n%s\n---SKIP---\n%s\n---SYS---\n%s\n---REBOOT---\n%s\n---RESTART---\n%s\n---STORAGE---\n%s\n---NETWORK---\n%s\n' "
. '"$(cat /proc/uptime|cut -d\" \" -f1)" '
. '"$(cat /proc/loadavg|cut -d\" \" -f1)" '
. '"$(nproc)" '
. '"$(grep -m1 MemTotal /proc/meminfo|awk \"{print \\\$2}\")" '
. '"$(grep -m1 MemAvailable /proc/meminfo|awk \"{print \\\$2}\")" '
. '"$(docker info >/dev/null 2>&1 && echo ok || echo err)" '
. '"$(cat /tmp/system_watchdog_oom.db 2>/dev/null||echo 0)" '
. '"$(wc -l < /tmp/watchdog_appdata_growth.db 2>/dev/null||echo 0)" '
. '"$(stat -c %Y /tmp/watchdog_appdata_growth.db 2>/dev/null||echo 0)" '
. '"$(cat /proc/meminfo 2>/dev/null)" '
. '"$(cat /tmp/resource_watchdog_state.db 2>/dev/null)" '
. '"$(cat /tmp/container_watchdog_state.db 2>/dev/null)" '
. '"$(cat /boot/config/system_watchdog_failed.db 2>/dev/null)" '
. '"$(cat /tmp/system_watchdog_state.db 2>/dev/null)" '
. '"$(cat /boot/config/system_watchdog_reboots.db 2>/dev/null)" '
. '"$(cat ' . escapeshellarg($restartLogPath) . ' 2>/dev/null)"';
. '"$(cat ' . escapeshellarg($restartLogPath) . ' 2>/dev/null)" '
. '"$(cat /tmp/storage_watchdog_state.db 2>/dev/null)" '
. '"$(cat /tmp/network_watchdog_state.db 2>/dev/null)"';
$out = vv_pt_ssh($ip, $sshKey, $cmd, 8);
if (!$out) return null;
// Parse sections
$sections = preg_split('/^---\w+---$/m', $out);
$header = $sections[0] ?? '';
$rwRaw = $sections[1] ?? '';
$dockRaw = $sections[2] ?? '';
$skipRaw = $sections[3] ?? '';
$sysRaw = $sections[4] ?? '';
$rebootRaw= $sections[5] ?? '';
$restartRaw=$sections[6] ?? '';
$sections = preg_split('/^---\w+---$/m', $out);
$header = $sections[0] ?? '';
$memInfoRaw = $sections[1] ?? '';
$rwRaw = $sections[2] ?? '';
$dockRaw = $sections[3] ?? '';
$skipRaw = $sections[4] ?? '';
$sysRaw = $sections[5] ?? '';
$rebootRaw = $sections[6] ?? '';
$restartRaw = $sections[7] ?? '';
$storRaw = $sections[8] ?? '';
$netWdRaw = $sections[9] ?? '';
// Parse header lines
$hdr = [];
@@ -199,6 +231,11 @@ function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath):
if (preg_match('/^(\w+):(.*)$/', trim($line), $m)) $hdr[$m[1]] = trim($m[2]);
}
// Parse /proc/meminfo section — no awk, no quoting issues
$memTotal = 0; $memAvail = 0;
if (preg_match('/^MemTotal:\s+(\d+)/m', $memInfoRaw, $m)) $memTotal = (int)$m[1] * 1024;
if (preg_match('/^MemAvailable:\s+(\d+)/m', $memInfoRaw, $m)) $memAvail = (int)$m[1] * 1024;
$rw = vv_wd_parse_kv($rwRaw);
$dock = vv_wd_parse_kv($dockRaw);
$sys = vv_wd_parse_kv($sysRaw);
@@ -213,8 +250,9 @@ function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath):
if (!str_contains($k, '=') && (int)$v > 0) $sysStrikes[$k] = (int)$v;
}
$memTotal = (int)($hdr['MEMTOTAL'] ?? 0) * 1024;
$memAvail = (int)($hdr['MEMAVAIL'] ?? 0) * 1024;
$baselineCount = (int)($hdr['BASELINECOUNT'] ?? 0);
$baselineTs = (int)($hdr['BASELINEAGE'] ?? 0);
$baselineAgeSec = $baselineTs > 0 ? time() - $baselineTs : null;
return [
'system' => [
@@ -239,6 +277,11 @@ function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath):
'sys_strikes' => $sysStrikes,
'reboots' => vv_wd_parse_reboot_log($rebootRaw),
'restarts' => vv_wd_parse_restart_log($restartRaw),
'storage_wd' => vv_wd_parse_storage_state($storRaw) + [
'baseline_count' => $baselineCount,
'baseline_age_sec' => $baselineAgeSec,
],
'network_wd' => vv_wd_parse_network_state($netWdRaw),
],
];
}
@@ -248,13 +291,19 @@ function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath):
function vv_wd_node_config(string $slot, string $raw, string $masterRaw): array {
$id = strtoupper($slot);
return [
'monitored' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINERS"),
'urls' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINER_URLS"),
'required' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_REQUIRED_CONTAINERS"),
'ignore' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_SCAN_IGNORE"),
'pause_list' => vv_wd_bash_array($raw, "{$id}_RW_PAUSE_CONTAINERS"),
'stop_list' => vv_wd_bash_array($raw, "{$id}_RW_STOP_CONTAINERS"),
'critical' => vv_wd_bash_array($masterRaw, "RW_CRITICAL_CONTAINERS"),
'monitored' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINERS"),
'urls' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINER_URLS"),
'required' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_REQUIRED_CONTAINERS"),
'ignore' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_SCAN_IGNORE"),
'pause_list' => vv_wd_bash_array($raw, "{$id}_RW_PAUSE_CONTAINERS"),
'stop_list' => vv_wd_bash_array($raw, "{$id}_RW_STOP_CONTAINERS"),
'critical' => vv_wd_bash_array($masterRaw, "RW_CRITICAL_CONTAINERS"),
// Network watchdog — host-specific
'ddns_domain' => vv_wd_scalar($raw, "{$id}_NETWORK_WATCHDOG_DDNS_DOMAIN"),
'ddns_container' => vv_wd_scalar($raw, "{$id}_NETWORK_WATCHDOG_DDNS_CONTAINER"),
'npm_url' => vv_wd_scalar($raw, "{$id}_NETWORK_WATCHDOG_NPM_URL"),
// Storage watchdog — host-specific appdata suppress ceilings
'appdata_sizes' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_APPDATA_SIZES"),
];
}
@@ -283,8 +332,17 @@ function vv_wd_all(): array {
'soft_mem_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_MEM_THRESHOLD') ?: 80),
'soft_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_CPU_THRESHOLD') ?: 80),
'hard_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'HARD_CPU_THRESHOLD') ?: 85),
'cpu_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'CPU_FAIL_LIMIT') ?: 2),
'resp_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'RESP_FAIL_LIMIT') ?: 2),
'cpu_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'CPU_FAIL_LIMIT') ?: 2),
'resp_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'RESP_FAIL_LIMIT') ?: 2),
// Storage watchdog
'growth_gb' => (float)(vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_GROWTH_GB') ?: 2),
'log_max_gb' => (float)(vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_LOG_MAX_GB') ?: 2),
'stor_strike_lim'=> (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_STRIKE_LIMIT') ?: 3),
'truncate_logs' => vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_TRUNCATE_LOGS') === 'true',
// Network watchdog
'net_wd_enabled' => vv_wd_scalar($masterRaw, 'NETWORK_WATCHDOG_ENABLED') !== 'false',
'npm_strike_lim' => (int)(vv_wd_scalar($masterRaw, 'NETWORK_WATCHDOG_NPM_STRIKE_LIMIT') ?: 2),
'ts_check' => vv_wd_scalar($masterRaw, 'NETWORK_WATCHDOG_CHECK_TAILSCALE') !== 'false',
];
// SSH key from current host conf
@@ -309,6 +367,8 @@ function vv_wd_all(): array {
$ip = $ts['ip'] ?? null;
$raw = vv_read_conf_raw($slot . '.conf');
$remoteApiKey = vv_wd_scalar($raw, strtoupper($slot) . '_UNRAID_API_KEY');
if ($isMe) {
$system = vv_wd_local_system();
$states = vv_wd_local_states($restartLog);
@@ -316,6 +376,28 @@ function vv_wd_all(): array {
$remote = vv_wd_remote_data($ip, $mySshKey, $restartLog);
$system = $remote['system'] ?? null;
$states = $remote['states'] ?? null;
} elseif ($remoteApiKey) {
$remoteStats = vv_remote_hosts_stats();
$rs = $remoteStats[strtoupper($slot)] ?? null;
if ($rs && ($rs['available'] ?? false)) {
$totalGb = (float)($rs['mem_total_gb'] ?? 0);
$usedPct = (float)($rs['mem_used_pct'] ?? 0) / 100;
$totalB = (int)($totalGb * 1073741824);
$availB = (int)($totalB * (1 - $usedPct));
$system = [
'mem_total' => $totalB,
'mem_avail' => $availB,
'load1' => 0.0,
'cores' => (int)($rs['cpu_threads'] ?? 0),
'uptime' => (int)($rs['uptime_sec'] ?? 0),
'daemon_ok' => null,
'oom_count' => 0,
'api_only' => true,
];
} else {
$system = null;
}
$states = null;
} else {
$system = null;
$states = null;