$k) $out[$k] = $pairs[2][$i]; return $out; } function vv_wd_scalar(string $raw, string $varname): string { return preg_match('/^\s*' . preg_quote($varname, '/') . '\s*=\s*"?([^"\n]*)"?/m', $raw, $m) ? trim($m[1]) : ''; } // ── State file parsers ──────────────────────────────────────────────────────── // Parses key:value format (with optional key=value mixed in) function vv_wd_parse_kv(string $text): array { $out = []; foreach (explode("\n", $text) as $line) { $line = trim($line); if (!$line) continue; if (str_contains($line, ':')) { [$k, $v] = explode(':', $line, 2); $out[trim($k)] = trim($v); } elseif (str_contains($line, '=')) { [$k, $v] = explode('=', $line, 2); $out[trim($k)] = trim($v, '"\''); } } return $out; } // Restart log: "container|timestamp" one per line function vv_wd_parse_restart_log(string $text, int $windowSeconds = 86400): array { $now = time(); $cutoff = $now - $windowSeconds; $entries = []; foreach (explode("\n", trim($text)) as $line) { $line = trim($line); if (!$line || !str_contains($line, '|')) continue; [$name, $ts] = explode('|', $line, 2); $ts = (int)$ts; if ($ts >= $cutoff) $entries[] = ['name' => trim($name), 'ts' => $ts]; } usort($entries, fn($a, $b) => $b['ts'] - $a['ts']); return $entries; } // Skip list: one container name per line function vv_wd_parse_skiplist(string $text): array { return array_values(array_filter(array_map('trim', explode("\n", $text)))); } // Reboot log: one timestamp per line function vv_wd_parse_reboot_log(string $text, int $windowHrs = 12): array { $cutoff = time() - ($windowHrs * 3600); $entries = []; foreach (explode("\n", trim($text)) as $line) { $ts = (int)trim($line); if ($ts > 0 && $ts >= $cutoff) $entries[] = $ts; } rsort($entries); return $entries; } // ── Local system snapshot ───────────────────────────────────────────────────── function vv_wd_local_system(): array { // RAM $memRaw = file_exists('/proc/meminfo') ? file_get_contents('/proc/meminfo') : ''; $memTotal = 0; $memAvail = 0; if (preg_match('/^MemTotal:\s+(\d+)/m', $memRaw, $m)) $memTotal = (int)$m[1] * 1024; if (preg_match('/^MemAvailable:\s+(\d+)/m', $memRaw, $m)) $memAvail = (int)$m[1] * 1024; // Load + cores $loadRaw = file_exists('/proc/loadavg') ? file_get_contents('/proc/loadavg') : '0 0 0'; $loadParts = explode(' ', trim($loadRaw)); $load1 = (float)($loadParts[0] ?? 0); $cores = (int)(trim(shell_exec('nproc 2>/dev/null') ?: '1')); // Uptime $uptimeRaw = file_exists('/proc/uptime') ? file_get_contents('/proc/uptime') : '0'; $uptime = (int)explode(' ', $uptimeRaw)[0]; // Docker daemon alive $daemonOk = (trim(shell_exec('docker info >/dev/null 2>&1; echo $?') ?: '1') === '0'); // OOM count (from stability watchdog OOM file — just the prev cycle count) $oomFile = '/tmp/system_watchdog_oom.db'; $oomCount = file_exists($oomFile) ? (int)trim(file_get_contents($oomFile)) : 0; return [ 'mem_total' => $memTotal, 'mem_avail' => $memAvail, 'load1' => $load1, 'cores' => $cores, 'uptime' => $uptime, 'daemon_ok' => $daemonOk, 'oom_count' => $oomCount, ]; } // ── Local state files ───────────────────────────────────────────────────────── function vv_wd_local_states(string $restartLogPath): array { $rwRaw = @file_get_contents('/tmp/resource_watchdog_state.db') ?: ''; $dockRaw = @file_get_contents('/tmp/container_watchdog_state.db') ?: ''; $skipRaw = @file_get_contents('/boot/config/system_watchdog_failed.db') ?: ''; $sysRaw = @file_get_contents('/tmp/system_watchdog_state.db') ?: ''; $rebootRaw = @file_get_contents('/boot/config/system_watchdog_reboots.db')?: ''; $restartRaw= @file_get_contents($restartLogPath) ?: ''; $rw = vv_wd_parse_kv($rwRaw); $dock = vv_wd_parse_kv($dockRaw); $sys = vv_wd_parse_kv($sysRaw); // Container strikes: everything in docker state that isn't a flag $strikes = []; foreach ($dock as $k => $v) { if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0) $strikes[$k] = (int)$v; } // Stability strikes: everything in sys state that isn't a flag key $sysStrikes = []; foreach ($sys as $k => $v) { if (!str_contains($k, '=') && (int)$v > 0) $sysStrikes[$k] = (int)$v; } return [ 'rw_level' => (int)($rw['rm_action_level'] ?? 0), 'rw_recover' => (int)($rw['rm_recover_cycles'] ?? 0), 'rw_paused' => array_filter(explode(',', $rw['rm_paused_containers'] ?? '')), 'rw_stopped' => array_filter(explode(',', $rw['rm_stopped_containers'] ?? '')), 'mem_shutdown' => ($rw['mem_shutdown_active'] ?? 'false') === 'true', 'daemon_strikes' => (int)($dock['daemon_strikes'] ?? 0), 'daemon_restart' => ($dock['daemon_restarted_flag'] ?? 'false') === 'true', 'ctr_strikes' => $strikes, 'skiplist' => vv_wd_parse_skiplist($skipRaw), 'sys_strikes' => $sysStrikes, 'reboots' => vv_wd_parse_reboot_log($rebootRaw), 'restarts' => vv_wd_parse_restart_log($restartRaw), ]; } // ── Remote data via SSH ─────────────────────────────────────────────────────── function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath): array { // Bundle into one SSH call $cmd = "printf 'UPTIME:%s\nLOAD:%s\nCORES:%s\nMEMTOTAL:%s\nMEMAVAIL:%s\nDAEMON:%s\nOOM:%s\n---RW---\n%s\n---DOCK---\n%s\n---SKIP---\n%s\n---SYS---\n%s\n---REBOOT---\n%s\n---RESTART---\n%s\n' " . '"$(cat /proc/uptime|cut -d\" \" -f1)" ' . '"$(cat /proc/loadavg|cut -d\" \" -f1)" ' . '"$(nproc)" ' . '"$(grep -m1 MemTotal /proc/meminfo|awk \"{print \\\$2}\")" ' . '"$(grep -m1 MemAvailable /proc/meminfo|awk \"{print \\\$2}\")" ' . '"$(docker info >/dev/null 2>&1 && echo ok || echo err)" ' . '"$(cat /tmp/system_watchdog_oom.db 2>/dev/null||echo 0)" ' . '"$(cat /tmp/resource_watchdog_state.db 2>/dev/null)" ' . '"$(cat /tmp/container_watchdog_state.db 2>/dev/null)" ' . '"$(cat /boot/config/system_watchdog_failed.db 2>/dev/null)" ' . '"$(cat /tmp/system_watchdog_state.db 2>/dev/null)" ' . '"$(cat /boot/config/system_watchdog_reboots.db 2>/dev/null)" ' . '"$(cat ' . escapeshellarg($restartLogPath) . ' 2>/dev/null)"'; $out = vv_pt_ssh($ip, $sshKey, $cmd, 8); if (!$out) return null; // Parse sections $sections = preg_split('/^---\w+---$/m', $out); $header = $sections[0] ?? ''; $rwRaw = $sections[1] ?? ''; $dockRaw = $sections[2] ?? ''; $skipRaw = $sections[3] ?? ''; $sysRaw = $sections[4] ?? ''; $rebootRaw= $sections[5] ?? ''; $restartRaw=$sections[6] ?? ''; // Parse header lines $hdr = []; foreach (explode("\n", $header) as $line) { if (preg_match('/^(\w+):(.*)$/', trim($line), $m)) $hdr[$m[1]] = trim($m[2]); } $rw = vv_wd_parse_kv($rwRaw); $dock = vv_wd_parse_kv($dockRaw); $sys = vv_wd_parse_kv($sysRaw); $strikes = []; foreach ($dock as $k => $v) { if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0) $strikes[$k] = (int)$v; } $sysStrikes = []; foreach ($sys as $k => $v) { if (!str_contains($k, '=') && (int)$v > 0) $sysStrikes[$k] = (int)$v; } $memTotal = (int)($hdr['MEMTOTAL'] ?? 0) * 1024; $memAvail = (int)($hdr['MEMAVAIL'] ?? 0) * 1024; return [ 'system' => [ 'mem_total' => $memTotal, 'mem_avail' => $memAvail, 'load1' => (float)($hdr['LOAD'] ?? 0), 'cores' => (int)($hdr['CORES'] ?? 1), 'uptime' => (int)($hdr['UPTIME'] ?? 0), 'daemon_ok' => ($hdr['DAEMON'] ?? '') === 'ok', 'oom_count' => (int)($hdr['OOM'] ?? 0), ], 'states' => [ 'rw_level' => (int)($rw['rm_action_level'] ?? 0), 'rw_recover' => (int)($rw['rm_recover_cycles'] ?? 0), 'rw_paused' => array_filter(explode(',', $rw['rm_paused_containers'] ?? '')), 'rw_stopped' => array_filter(explode(',', $rw['rm_stopped_containers'] ?? '')), 'mem_shutdown' => ($rw['mem_shutdown_active'] ?? 'false') === 'true', 'daemon_strikes'=> (int)($dock['daemon_strikes'] ?? 0), 'daemon_restart'=> ($dock['daemon_restarted_flag'] ?? 'false') === 'true', 'ctr_strikes' => $strikes, 'skiplist' => vv_wd_parse_skiplist($skipRaw), 'sys_strikes' => $sysStrikes, 'reboots' => vv_wd_parse_reboot_log($rebootRaw), 'restarts' => vv_wd_parse_restart_log($restartRaw), ], ]; } // ── Config inventory ────────────────────────────────────────────────────────── function vv_wd_node_config(string $slot, string $raw, string $masterRaw): array { $id = strtoupper($slot); return [ 'monitored' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINERS"), 'urls' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINER_URLS"), 'required' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_REQUIRED_CONTAINERS"), 'ignore' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_SCAN_IGNORE"), 'pause_list' => vv_wd_bash_array($raw, "{$id}_RW_PAUSE_CONTAINERS"), 'stop_list' => vv_wd_bash_array($raw, "{$id}_RW_STOP_CONTAINERS"), 'critical' => vv_wd_bash_array($masterRaw, "RW_CRITICAL_CONTAINERS"), ]; } // ── Entry point ─────────────────────────────────────────────────────────────── function vv_wd_all(): array { $currentHost = vv_detect_host(); $tsPeers = vv_pt_ts_peers(); $masterRaw = vv_read_conf_raw('master.conf'); $restartLog = '/mnt/user/appdata/unraid_scripts/data/container_restart_history.db'; // Config thresholds from master.conf $cfg = [ 'rw_soft_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_SOFT_GB') ?: 12), 'rw_medium_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_MEDIUM_GB') ?: 8), 'rw_hard_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_HARD_GB') ?: 6), 'rw_recover_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_RECOVER_GB') ?: 20), 'rw_load_soft' => (float)(vv_wd_scalar($masterRaw, 'RW_LOAD_SOFT_MULTIPLIER') ?: 2.0), 'rw_load_med' => (float)(vv_wd_scalar($masterRaw, 'RW_LOAD_MEDIUM_MULTIPLIER') ?: 3.0), 'sys_mem_gb' => (float)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_MEM_GB') ?: 4), 'sys_strikes' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_STRIKE_LIMIT') ?: 2), 'reboot_limit' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_REBOOT_LIMIT') ?: 3), 'reboot_window' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_REBOOT_WINDOW_HRS') ?: 12), 'restart_limit' => (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_CONTAINER_RESTART_LIMIT') ?: 3), 'startup_grace' => (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_STARTUP_GRACE') ?: 600), 'soft_mem_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_MEM_THRESHOLD') ?: 80), 'soft_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_CPU_THRESHOLD') ?: 80), 'hard_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'HARD_CPU_THRESHOLD') ?: 85), 'cpu_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'CPU_FAIL_LIMIT') ?: 2), 'resp_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'RESP_FAIL_LIMIT') ?: 2), ]; // SSH key from current host conf $myId = strtoupper($currentHost); $myRaw = vv_read_conf_raw($currentHost . '.conf'); $mySshKey = vv_wd_scalar($myRaw, $myId . '_SSH_KEY'); // Known hosts preg_match_all('/^\s*(HOST(\d+))(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $masterRaw, $m); $hosts = []; foreach ($m[1] as $i => $key) { $hosts['host' . $m[2][$i]] = trim($m[3][$i]); } ksort($hosts); if (!$hosts) $hosts = ['host1' => 'HOST1']; $nodes = []; foreach ($hosts as $slot => $hostname) { $isMe = ($slot === $currentHost || $currentHost === 'unknown'); $tsLabel = strtolower($hostname); $ts = $tsPeers[$tsLabel] ?? ['online' => null, 'active' => false, 'ip' => null]; $ip = $ts['ip'] ?? null; $raw = vv_read_conf_raw($slot . '.conf'); if ($isMe) { $system = vv_wd_local_system(); $states = vv_wd_local_states($restartLog); } elseif ($ip && $mySshKey && $ts['online']) { $remote = vv_wd_remote_data($ip, $mySshKey, $restartLog); $system = $remote['system'] ?? null; $states = $remote['states'] ?? null; } else { $system = null; $states = null; } $nodes[] = [ 'slot' => $slot, 'id' => strtoupper($slot), 'hostname' => $hostname, 'is_me' => $isMe, 'ts_online' => $ts['online'], 'system' => $system, 'states' => $states, 'config' => vv_wd_node_config($slot, $raw, $masterRaw), ]; } return [ 'ts' => time(), 'cfg' => $cfg, 'nodes' => $nodes, ]; }