FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Varaverk: FallBack + Watchdog tabs; plugin path restructure to Plugin/unraid/

Browse Source 
- FallBack tab: per-node tier inventory + active fallback card with duration, tier, handback strikes, running container status
- Watchdog tab: live system health (RAM bar + thresholds, load, uptime, daemon), docker watchdog strikes + skip list + restart history, stability strikes + reboot log, resource pressure alert card, config inventory (mem limits, required, pause/stop lists)
- Swapped partnership/arrs tab order; FallBack between partnership and watchdog
- Plugin source tree moved from Plugin/usr/local/emhttp/plugins/varaverk/ to Plugin/unraid/
- Deployment/ conf templates added
This commit is contained in:
Gmer4Lfe
2026-05-28 22:24:50 -04:00
parent 64d95aa991
commit fb051b60c1
73 changed files with 5896 additions and 462 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Plugin/unraid/include/watchdog.php
+341
View File
@@ -0,0 +1,341 @@
<?php
// Watchdog tab data helpers
require_once __DIR__ . '/config.php';
require_once __DIR__ . '/partnership.php'; // vv_pt_ssh(), vv_pt_ts_peers()
// ── Conf array parser (bash arrays) ──────────────────────────────────────────
function vv_wd_bash_array(string $raw, string $varname): array {
if (!preg_match('/^\s*' . preg_quote($varname, '/') . '\s*=\s*\(\s*(.*?)\s*\)/ms', $raw, $m))
return [];
preg_match_all('/"([^"]*)"/', $m[1], $items);
return array_values(array_filter($items[1]));
}
function vv_wd_bash_assoc(string $raw, string $varname): array {
// declare -A VARNAME=( ["key"]=val ["key2"]=val2 )
if (!preg_match('/^\s*declare\s+-A\s+' . preg_quote($varname, '/') . '\s*=\s*\(\s*(.*?)\s*\)/ms', $raw, $m))
return [];
preg_match_all('/\["([^"]+)"\]\s*=\s*"?([^"\s\)]*)"?/', $m[1], $pairs);
$out = [];
foreach ($pairs[1] as $i => $k) $out[$k] = $pairs[2][$i];
return $out;
}
function vv_wd_scalar(string $raw, string $varname): string {
return preg_match('/^\s*' . preg_quote($varname, '/') . '\s*=\s*"?([^"\n]*)"?/m', $raw, $m)
? trim($m[1]) : '';
}
// ── State file parsers ────────────────────────────────────────────────────────
// Parses key:value format (with optional key=value mixed in)
function vv_wd_parse_kv(string $text): array {
$out = [];
foreach (explode("\n", $text) as $line) {
$line = trim($line);
if (!$line) continue;
if (str_contains($line, ':')) {
[$k, $v] = explode(':', $line, 2);
$out[trim($k)] = trim($v);
} elseif (str_contains($line, '=')) {
[$k, $v] = explode('=', $line, 2);
$out[trim($k)] = trim($v, '"\'');
}
}
return $out;
}
// Restart log: "container|timestamp" one per line
function vv_wd_parse_restart_log(string $text, int $windowSeconds = 86400): array {
$now = time();
$cutoff = $now - $windowSeconds;
$entries = [];
foreach (explode("\n", trim($text)) as $line) {
$line = trim($line);
if (!$line || !str_contains($line, '|')) continue;
[$name, $ts] = explode('|', $line, 2);
$ts = (int)$ts;
if ($ts >= $cutoff) $entries[] = ['name' => trim($name), 'ts' => $ts];
}
usort($entries, fn($a, $b) => $b['ts'] - $a['ts']);
return $entries;
}
// Skip list: one container name per line
function vv_wd_parse_skiplist(string $text): array {
return array_values(array_filter(array_map('trim', explode("\n", $text))));
}
// Reboot log: one timestamp per line
function vv_wd_parse_reboot_log(string $text, int $windowHrs = 12): array {
$cutoff = time() - ($windowHrs * 3600);
$entries = [];
foreach (explode("\n", trim($text)) as $line) {
$ts = (int)trim($line);
if ($ts > 0 && $ts >= $cutoff) $entries[] = $ts;
}
rsort($entries);
return $entries;
}
// ── Local system snapshot ─────────────────────────────────────────────────────
function vv_wd_local_system(): array {
// RAM
$memRaw = file_exists('/proc/meminfo') ? file_get_contents('/proc/meminfo') : '';
$memTotal = 0; $memAvail = 0;
if (preg_match('/^MemTotal:\s+(\d+)/m', $memRaw, $m)) $memTotal = (int)$m[1] * 1024;
if (preg_match('/^MemAvailable:\s+(\d+)/m', $memRaw, $m)) $memAvail = (int)$m[1] * 1024;
// Load + cores
$loadRaw = file_exists('/proc/loadavg') ? file_get_contents('/proc/loadavg') : '0 0 0';
$loadParts = explode(' ', trim($loadRaw));
$load1 = (float)($loadParts[0] ?? 0);
$cores = (int)(trim(shell_exec('nproc 2>/dev/null') ?: '1'));
// Uptime
$uptimeRaw = file_exists('/proc/uptime') ? file_get_contents('/proc/uptime') : '0';
$uptime = (int)explode(' ', $uptimeRaw)[0];
// Docker daemon alive
$daemonOk = (trim(shell_exec('docker info >/dev/null 2>&1; echo $?') ?: '1') === '0');
// OOM count (from stability watchdog OOM file — just the prev cycle count)
$oomFile = '/tmp/system_watchdog_oom.db';
$oomCount = file_exists($oomFile) ? (int)trim(file_get_contents($oomFile)) : 0;
return [
'mem_total' => $memTotal,
'mem_avail' => $memAvail,
'load1' => $load1,
'cores' => $cores,
'uptime' => $uptime,
'daemon_ok' => $daemonOk,
'oom_count' => $oomCount,
];
}
// ── Local state files ─────────────────────────────────────────────────────────
function vv_wd_local_states(string $restartLogPath): array {
$rwRaw = @file_get_contents('/tmp/resource_watchdog_state.db') ?: '';
$dockRaw = @file_get_contents('/tmp/container_watchdog_state.db') ?: '';
$skipRaw = @file_get_contents('/boot/config/system_watchdog_failed.db') ?: '';
$sysRaw = @file_get_contents('/tmp/system_watchdog_state.db') ?: '';
$rebootRaw = @file_get_contents('/boot/config/system_watchdog_reboots.db')?: '';
$restartRaw= @file_get_contents($restartLogPath) ?: '';
$rw = vv_wd_parse_kv($rwRaw);
$dock = vv_wd_parse_kv($dockRaw);
$sys = vv_wd_parse_kv($sysRaw);
// Container strikes: everything in docker state that isn't a flag
$strikes = [];
foreach ($dock as $k => $v) {
if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0)
$strikes[$k] = (int)$v;
}
// Stability strikes: everything in sys state that isn't a flag key
$sysStrikes = [];
foreach ($sys as $k => $v) {
if (!str_contains($k, '=') && (int)$v > 0)
$sysStrikes[$k] = (int)$v;
}
return [
'rw_level' => (int)($rw['rm_action_level'] ?? 0),
'rw_recover' => (int)($rw['rm_recover_cycles'] ?? 0),
'rw_paused' => array_filter(explode(',', $rw['rm_paused_containers'] ?? '')),
'rw_stopped' => array_filter(explode(',', $rw['rm_stopped_containers'] ?? '')),
'mem_shutdown' => ($rw['mem_shutdown_active'] ?? 'false') === 'true',
'daemon_strikes' => (int)($dock['daemon_strikes'] ?? 0),
'daemon_restart' => ($dock['daemon_restarted_flag'] ?? 'false') === 'true',
'ctr_strikes' => $strikes,
'skiplist' => vv_wd_parse_skiplist($skipRaw),
'sys_strikes' => $sysStrikes,
'reboots' => vv_wd_parse_reboot_log($rebootRaw),
'restarts' => vv_wd_parse_restart_log($restartRaw),
];
}
// ── Remote data via SSH ───────────────────────────────────────────────────────
function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath): array {
// Bundle into one SSH call
$cmd = "printf 'UPTIME:%s\nLOAD:%s\nCORES:%s\nMEMTOTAL:%s\nMEMAVAIL:%s\nDAEMON:%s\nOOM:%s\n---RW---\n%s\n---DOCK---\n%s\n---SKIP---\n%s\n---SYS---\n%s\n---REBOOT---\n%s\n---RESTART---\n%s\n' "
. '"$(cat /proc/uptime|cut -d\" \" -f1)" '
. '"$(cat /proc/loadavg|cut -d\" \" -f1)" '
. '"$(nproc)" '
. '"$(grep -m1 MemTotal /proc/meminfo|awk \"{print \\\$2}\")" '
. '"$(grep -m1 MemAvailable /proc/meminfo|awk \"{print \\\$2}\")" '
. '"$(docker info >/dev/null 2>&1 && echo ok || echo err)" '
. '"$(cat /tmp/system_watchdog_oom.db 2>/dev/null||echo 0)" '
. '"$(cat /tmp/resource_watchdog_state.db 2>/dev/null)" '
. '"$(cat /tmp/container_watchdog_state.db 2>/dev/null)" '
. '"$(cat /boot/config/system_watchdog_failed.db 2>/dev/null)" '
. '"$(cat /tmp/system_watchdog_state.db 2>/dev/null)" '
. '"$(cat /boot/config/system_watchdog_reboots.db 2>/dev/null)" '
. '"$(cat ' . escapeshellarg($restartLogPath) . ' 2>/dev/null)"';
$out = vv_pt_ssh($ip, $sshKey, $cmd, 8);
if (!$out) return null;
// Parse sections
$sections = preg_split('/^---\w+---$/m', $out);
$header = $sections[0] ?? '';
$rwRaw = $sections[1] ?? '';
$dockRaw = $sections[2] ?? '';
$skipRaw = $sections[3] ?? '';
$sysRaw = $sections[4] ?? '';
$rebootRaw= $sections[5] ?? '';
$restartRaw=$sections[6] ?? '';
// Parse header lines
$hdr = [];
foreach (explode("\n", $header) as $line) {
if (preg_match('/^(\w+):(.*)$/', trim($line), $m)) $hdr[$m[1]] = trim($m[2]);
}
$rw = vv_wd_parse_kv($rwRaw);
$dock = vv_wd_parse_kv($dockRaw);
$sys = vv_wd_parse_kv($sysRaw);
$strikes = [];
foreach ($dock as $k => $v) {
if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0)
$strikes[$k] = (int)$v;
}
$sysStrikes = [];
foreach ($sys as $k => $v) {
if (!str_contains($k, '=') && (int)$v > 0) $sysStrikes[$k] = (int)$v;
}
$memTotal = (int)($hdr['MEMTOTAL'] ?? 0) * 1024;
$memAvail = (int)($hdr['MEMAVAIL'] ?? 0) * 1024;
return [
'system' => [
'mem_total' => $memTotal,
'mem_avail' => $memAvail,
'load1' => (float)($hdr['LOAD'] ?? 0),
'cores' => (int)($hdr['CORES'] ?? 1),
'uptime' => (int)($hdr['UPTIME'] ?? 0),
'daemon_ok' => ($hdr['DAEMON'] ?? '') === 'ok',
'oom_count' => (int)($hdr['OOM'] ?? 0),
],
'states' => [
'rw_level' => (int)($rw['rm_action_level'] ?? 0),
'rw_recover' => (int)($rw['rm_recover_cycles'] ?? 0),
'rw_paused' => array_filter(explode(',', $rw['rm_paused_containers'] ?? '')),
'rw_stopped' => array_filter(explode(',', $rw['rm_stopped_containers'] ?? '')),
'mem_shutdown' => ($rw['mem_shutdown_active'] ?? 'false') === 'true',
'daemon_strikes'=> (int)($dock['daemon_strikes'] ?? 0),
'daemon_restart'=> ($dock['daemon_restarted_flag'] ?? 'false') === 'true',
'ctr_strikes' => $strikes,
'skiplist' => vv_wd_parse_skiplist($skipRaw),
'sys_strikes' => $sysStrikes,
'reboots' => vv_wd_parse_reboot_log($rebootRaw),
'restarts' => vv_wd_parse_restart_log($restartRaw),
],
];
}
// ── Config inventory ──────────────────────────────────────────────────────────
function vv_wd_node_config(string $slot, string $raw, string $masterRaw): array {
$id = strtoupper($slot);
return [
'monitored' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINERS"),
'urls' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINER_URLS"),
'required' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_REQUIRED_CONTAINERS"),
'ignore' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_SCAN_IGNORE"),
'pause_list' => vv_wd_bash_array($raw, "{$id}_RW_PAUSE_CONTAINERS"),
'stop_list' => vv_wd_bash_array($raw, "{$id}_RW_STOP_CONTAINERS"),
'critical' => vv_wd_bash_array($masterRaw, "RW_CRITICAL_CONTAINERS"),
];
}
// ── Entry point ───────────────────────────────────────────────────────────────
function vv_wd_all(): array {
$currentHost = vv_detect_host();
$tsPeers = vv_pt_ts_peers();
$masterRaw = vv_read_conf_raw('master.conf');
$restartLog = '/mnt/user/appdata/unraid_scripts/data/container_restart_history.db';
// Config thresholds from master.conf
$cfg = [
'rw_soft_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_SOFT_GB') ?: 12),
'rw_medium_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_MEDIUM_GB') ?: 8),
'rw_hard_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_HARD_GB') ?: 6),
'rw_recover_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_RECOVER_GB') ?: 20),
'rw_load_soft' => (float)(vv_wd_scalar($masterRaw, 'RW_LOAD_SOFT_MULTIPLIER') ?: 2.0),
'rw_load_med' => (float)(vv_wd_scalar($masterRaw, 'RW_LOAD_MEDIUM_MULTIPLIER') ?: 3.0),
'sys_mem_gb' => (float)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_MEM_GB') ?: 4),
'sys_strikes' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_STRIKE_LIMIT') ?: 2),
'reboot_limit' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_REBOOT_LIMIT') ?: 3),
'reboot_window' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_REBOOT_WINDOW_HRS') ?: 12),
'restart_limit' => (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_CONTAINER_RESTART_LIMIT') ?: 3),
'startup_grace' => (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_STARTUP_GRACE') ?: 600),
'soft_mem_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_MEM_THRESHOLD') ?: 80),
'soft_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_CPU_THRESHOLD') ?: 80),
'hard_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'HARD_CPU_THRESHOLD') ?: 85),
'cpu_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'CPU_FAIL_LIMIT') ?: 2),
'resp_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'RESP_FAIL_LIMIT') ?: 2),
];
// SSH key from current host conf
$myId = strtoupper($currentHost);
$myRaw = vv_read_conf_raw($currentHost . '.conf');
$mySshKey = vv_wd_scalar($myRaw, $myId . '_SSH_KEY');
// Known hosts
preg_match_all('/^\s*(HOST(\d+))(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $masterRaw, $m);
$hosts = [];
foreach ($m[1] as $i => $key) {
$hosts['host' . $m[2][$i]] = trim($m[3][$i]);
}
ksort($hosts);
if (!$hosts) $hosts = ['host1' => 'HOST1'];
$nodes = [];
foreach ($hosts as $slot => $hostname) {
$isMe = ($slot === $currentHost || $currentHost === 'unknown');
$tsLabel = strtolower($hostname);
$ts = $tsPeers[$tsLabel] ?? ['online' => null, 'active' => false, 'ip' => null];
$ip = $ts['ip'] ?? null;
$raw = vv_read_conf_raw($slot . '.conf');
if ($isMe) {
$system = vv_wd_local_system();
$states = vv_wd_local_states($restartLog);
} elseif ($ip && $mySshKey && $ts['online']) {
$remote = vv_wd_remote_data($ip, $mySshKey, $restartLog);
$system = $remote['system'] ?? null;
$states = $remote['states'] ?? null;
} else {
$system = null;
$states = null;
}
$nodes[] = [
'slot' => $slot,
'id' => strtoupper($slot),
'hostname' => $hostname,
'is_me' => $isMe,
'ts_online' => $ts['online'],
'system' => $system,
'states' => $states,
'config' => vv_wd_node_config($slot, $raw, $masterRaw),
];
}
return [
'ts' => time(),
'cfg' => $cfg,
'nodes' => $nodes,
];
}