$k) $out[$k] = $pairs[2][$i]; return $out; } function vv_wd_scalar(string $raw, string $varname): string { return vv_parse_conf_scalar($raw, $varname); } // ── State file parsers ──────────────────────────────────────────────────────── // Parses key:value format (with optional key=value mixed in) function vv_wd_parse_kv(string $text): array { $out = []; foreach (explode("\n", $text) as $line) { $line = trim($line); if (!$line) continue; if (str_contains($line, ':')) { [$k, $v] = explode(':', $line, 2); $out[trim($k)] = trim($v); } elseif (str_contains($line, '=')) { [$k, $v] = explode('=', $line, 2); $out[trim($k)] = trim($v, '"\''); } } return $out; } // Restart log: "container|2026-08-02 13:15:07" one per line. // // The second field is a formatted local timestamp, not an epoch — docker_watchdog.sh writes it // with date '+%Y-%m-%d %H:%M:%S' because its own rolling-window trim compares the strings // lexically in awk, which is correct for that format. This function used to cast it with (int), // which stops at the first non-digit and yielded 2026 for every line ever written. 2026 is below // any plausible cutoff, so every entry was discarded and both restart panels — the Monitor card // and the Watchdog page — were permanently empty. Not visibly broken: an empty list reads // exactly like "nothing has restarted", which is the answer you least want to be wrong about // during a restart loop. // // Parsed, not reformatted. Changing what the watchdog writes would strand every existing entry // and every awk comparison in Tools/watchdog_skip_list_manager.sh. function vv_wd_parse_restart_log(string $text, int $windowSeconds = 86400): array { $cutoff = time() - $windowSeconds; $entries = []; foreach (explode("\n", trim($text)) as $line) { $line = trim($line); if (!$line || !str_contains($line, '|')) continue; [$name, $raw] = explode('|', $line, 2); $ts = vv_wd_restart_ts(trim($raw)); if ($ts >= $cutoff) $entries[] = ['name' => trim($name), 'ts' => $ts]; } usort($entries, fn($a, $b) => $b['ts'] - $a['ts']); return $entries; } // vv_wd_restart_ts() lives in common.php — include/monitor.php parses the same file and does not // include this one, so a helper defined here would be a fatal on the dashboard. // Skip list: one container name per line function vv_wd_parse_skiplist(string $text): array { return array_values(array_filter(array_map('trim', explode("\n", $text)))); } // Reboot log: one timestamp per line function vv_wd_parse_reboot_log(string $text, int $windowHrs = 12): array { $cutoff = time() - ($windowHrs * 3600); $entries = []; foreach (explode("\n", trim($text)) as $line) { $ts = (int)trim($line); if ($ts > 0 && $ts >= $cutoff) $entries[] = $ts; } rsort($entries); return $entries; } // ── Local system snapshot ───────────────────────────────────────────────────── function vv_wd_local_system(): array { $sys = vv_system_info(); $mem = vv_memory_breakdown(); $loadRaw = @file_get_contents('/proc/loadavg') ?: '0'; $daemonOk = (trim(shell_exec('docker info >/dev/null 2>&1; echo $?') ?: '1') === '0'); $oomCount = (int)trim(@file_get_contents(STATE_DIR . '/system_watchdog_oom.db') ?: '0'); $cores = (int)($sys['cpu_cores'] ?: (int)(trim(shell_exec('nproc 2>/dev/null') ?: '1'))); return [ 'mem_total' => (int)($mem['total_kb'] * 1024), 'mem_avail' => (int)($mem['free_kb'] * 1024), 'load1' => (float)explode(' ', trim($loadRaw))[0], 'cores' => $cores, 'uptime' => $sys['uptime_sec'] ?? 0, 'daemon_ok' => $daemonOk, 'oom_count' => $oomCount, ]; } // ── Storage / network state parsers ────────────────────────────────────────── function vv_wd_parse_storage_state(string $raw): array { $growth = []; $log = []; foreach (explode("\n", $raw) as $line) { $line = trim($line); if (!$line || !str_contains($line, ':')) continue; [$k, $v] = explode(':', $line, 2); $count = (int)trim($v); if ($count <= 0) continue; $key = trim($k); if (str_starts_with($key, 'appdata_growth_')) $growth[substr($key, strlen('appdata_growth_'))] = $count; elseif (str_starts_with($key, 'appdata_log_')) $log[substr($key, strlen('appdata_log_'))] = $count; } return ['growth_strikes' => $growth, 'log_strikes' => $log]; } function vv_wd_parse_network_state(string $raw): array { $npm = 0; foreach (explode("\n", $raw) as $line) { $line = trim($line); if (str_starts_with($line, 'npm:')) $npm = (int)trim(substr($line, 4)); } return ['npm_strikes' => $npm]; } // ── Local state files ───────────────────────────────────────────────────────── function vv_wd_local_states(string $restartLogPath): array { $rwRaw = @file_get_contents(STATE_DIR . '/resource_watchdog_state.db') ?: ''; $dockRaw = @file_get_contents(STATE_DIR . '/container_watchdog_state.db') ?: ''; $skipRaw = @file_get_contents(STATE_DIR . '/docker_watchdog_failed.db') ?: ''; $sysRaw = @file_get_contents(STATE_DIR . '/system_watchdog_state.db') ?: ''; $rebootRaw = @file_get_contents(STATE_DIR . '/system_watchdog_reboots.db') ?: ''; $restartRaw= @file_get_contents($restartLogPath) ?: ''; $storRaw = @file_get_contents(STATE_DIR . '/storage_watchdog_state.db') ?: ''; $netWdRaw = @file_get_contents(STATE_DIR . '/network_watchdog_state.db') ?: ''; $rw = vv_wd_parse_kv($rwRaw); $dock = vv_wd_parse_kv($dockRaw); $sys = vv_wd_parse_kv($sysRaw); // Container strikes: everything in docker state that isn't a flag $strikes = []; foreach ($dock as $k => $v) { if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0) $strikes[$k] = (int)$v; } // Stability strikes: everything in sys state that is actually a strike. // // The old guard was `!str_contains($k, '=')`, written on the belief that the bookkeeping rows // in system_watchdog_state.db keep their `=` inside the key because the file mixes separators // — `ram:0` and `sshd:0` beside `kernel_oops_count=0` and `watchdog_cycle=...`. They do not: // vv_wd_parse_kv() splits on both, so the guard never matched anything and every bookkeeping // row was published as a strike. watchdog_cycle is a unix timestamp written once per cycle by // stability_watchdog.sh:883 as a liveness heartbeat, so the Stability card was drawing // "watchdog cycle — 1786716931" next to a limit of 2. $sysStrikes = []; foreach ($sys as $k => $v) { if (in_array($k, VV_WD_SYS_BOOKKEEPING, true)) continue; if ((int)$v > 0) $sysStrikes[$k] = (int)$v; } // Growth baseline info (container count + age in seconds) $growthFile = STATE_DIR . '/watchdog_appdata_growth.db'; $baselineCount = file_exists($growthFile) ? max(0, count(file($growthFile)) - 0) : 0; $baselineAgeSec = file_exists($growthFile) ? time() - (int)filemtime($growthFile) : null; return [ 'rw_level' => (int)($rw['rm_action_level'] ?? 0), 'rw_recover' => (int)($rw['rm_recover_cycles'] ?? 0), 'rw_paused' => array_filter(explode(',', $rw['rm_paused_containers'] ?? '')), 'rw_stopped' => array_filter(explode(',', $rw['rm_stopped_containers'] ?? '')), 'mem_shutdown' => ($rw['mem_shutdown_active'] ?? 'false') === 'true', 'daemon_strikes' => (int)($dock['daemon_strikes'] ?? 0), 'daemon_restart' => ($dock['daemon_restarted_flag'] ?? 'false') === 'true', 'ctr_strikes' => $strikes, 'skiplist' => vv_wd_parse_skiplist($skipRaw), 'sys_strikes' => $sysStrikes, 'reboots' => vv_wd_parse_reboot_log($rebootRaw), 'restarts' => vv_wd_parse_restart_log($restartRaw), 'storage_wd' => vv_wd_parse_storage_state($storRaw) + [ 'baseline_count' => $baselineCount, 'baseline_age_sec' => $baselineAgeSec, ], 'network_wd' => vv_wd_parse_network_state($netWdRaw), ]; } // ── Remote data via SSH ─────────────────────────────────────────────────────── function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath): array { // Bundle into one SSH call. // /proc/meminfo is passed as a raw section (not awk-parsed) to avoid quoting // fragility — escapeshellarg() single-quotes the whole command so awk \$2 // inside double-quotes is unreliable across Unraid builds. // State files live under the REMOTE's own SCRIPTS_DIR (may differ from ours in flash mode), // so it is resolved on the far side — same idiom as vv_remote_state_cmd(). // // The layout is probed rather than assumed. State moved from SCRIPTS_DIR/State_Files to // DATA_DIR/state, and the histories from DATA_DIR's root into DATA_DIR/db, but a partner is // not guaranteed to have pulled that yet — and this is the call that reports whether the // partner's watchdogs are healthy. Guessing wrong returns empty strings for every state // file, which reads as "partner has no strikes" rather than as an error. Probing costs one // directory test and makes the answer correct in both directions, which also means the two // hosts can be upgraded in either order. $restartLogName = basename($restartLogPath); $cmd = 'sd=$(grep -m1 SCRIPTS_DIR= /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null' . ' | cut -d\'"\' -f2); sd="${sd:-/boot/config/plugins/varaverk}"; ' . 'sf="$sd/data/state"; [ -d "$sf" ] || sf="$sd/State_Files"; ' . 'db="$sd/data/db"; [ -d "$db" ] || db="$sd/data"; ' . "printf 'UPTIME:%s\nLOAD:%s\nCORES:%s\nDAEMON:%s\nOOM:%s\nBASELINECOUNT:%s\nBASELINEAGE:%s\n---MEMINFO---\n%s\n---RW---\n%s\n---DOCK---\n%s\n---SKIP---\n%s\n---SYS---\n%s\n---REBOOT---\n%s\n---RESTART---\n%s\n---STORAGE---\n%s\n---NETWORK---\n%s\n' " . '"$(cat /proc/uptime|cut -d\" \" -f1)" ' . '"$(cat /proc/loadavg|cut -d\" \" -f1)" ' . '"$(nproc)" ' . '"$(docker info >/dev/null 2>&1 && echo ok || echo err)" ' . '"$(cat "$sf/system_watchdog_oom.db" 2>/dev/null||echo 0)" ' . '"$(wc -l < "$sf/watchdog_appdata_growth.db" 2>/dev/null||echo 0)" ' . '"$(stat -c %Y "$sf/watchdog_appdata_growth.db" 2>/dev/null||echo 0)" ' . '"$(cat /proc/meminfo 2>/dev/null)" ' . '"$(cat "$sf/resource_watchdog_state.db" 2>/dev/null)" ' . '"$(cat "$sf/container_watchdog_state.db" 2>/dev/null)" ' . '"$(cat "$sf/docker_watchdog_failed.db" 2>/dev/null)" ' . '"$(cat "$sf/system_watchdog_state.db" 2>/dev/null)" ' . '"$(cat "$sf/system_watchdog_reboots.db" 2>/dev/null)" ' . '"$(cat "$db/' . $restartLogName . '" 2>/dev/null)" ' . '"$(cat "$sf/storage_watchdog_state.db" 2>/dev/null)" ' . '"$(cat "$sf/network_watchdog_state.db" 2>/dev/null)"'; $out = vv_pt_ssh($ip, $sshKey, $cmd, 8); if (!$out) return null; // Parse sections $sections = preg_split('/^---\w+---$/m', $out); $header = $sections[0] ?? ''; $memInfoRaw = $sections[1] ?? ''; $rwRaw = $sections[2] ?? ''; $dockRaw = $sections[3] ?? ''; $skipRaw = $sections[4] ?? ''; $sysRaw = $sections[5] ?? ''; $rebootRaw = $sections[6] ?? ''; $restartRaw = $sections[7] ?? ''; $storRaw = $sections[8] ?? ''; $netWdRaw = $sections[9] ?? ''; // Parse header lines $hdr = []; foreach (explode("\n", $header) as $line) { if (preg_match('/^(\w+):(.*)$/', trim($line), $m)) $hdr[$m[1]] = trim($m[2]); } // Parse /proc/meminfo section — no awk, no quoting issues $memTotal = 0; $memAvail = 0; if (preg_match('/^MemTotal:\s+(\d+)/m', $memInfoRaw, $m)) $memTotal = (int)$m[1] * 1024; if (preg_match('/^MemAvailable:\s+(\d+)/m', $memInfoRaw, $m)) $memAvail = (int)$m[1] * 1024; $rw = vv_wd_parse_kv($rwRaw); $dock = vv_wd_parse_kv($dockRaw); $sys = vv_wd_parse_kv($sysRaw); $strikes = []; foreach ($dock as $k => $v) { if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0) $strikes[$k] = (int)$v; } // Same exclusion as the local path above — a partner's heartbeat is no more a strike than // this host's. The two builders are maintained together. $sysStrikes = []; foreach ($sys as $k => $v) { if (in_array($k, VV_WD_SYS_BOOKKEEPING, true)) continue; if ((int)$v > 0) $sysStrikes[$k] = (int)$v; } $baselineCount = (int)($hdr['BASELINECOUNT'] ?? 0); $baselineTs = (int)($hdr['BASELINEAGE'] ?? 0); $baselineAgeSec = $baselineTs > 0 ? time() - $baselineTs : null; return [ 'system' => [ 'mem_total' => $memTotal, 'mem_avail' => $memAvail, 'load1' => (float)($hdr['LOAD'] ?? 0), 'cores' => (int)($hdr['CORES'] ?? 1), 'uptime' => (int)($hdr['UPTIME'] ?? 0), 'daemon_ok' => ($hdr['DAEMON'] ?? '') === 'ok', 'oom_count' => (int)($hdr['OOM'] ?? 0), ], 'states' => [ 'rw_level' => (int)($rw['rm_action_level'] ?? 0), 'rw_recover' => (int)($rw['rm_recover_cycles'] ?? 0), 'rw_paused' => array_filter(explode(',', $rw['rm_paused_containers'] ?? '')), 'rw_stopped' => array_filter(explode(',', $rw['rm_stopped_containers'] ?? '')), 'mem_shutdown' => ($rw['mem_shutdown_active'] ?? 'false') === 'true', 'daemon_strikes'=> (int)($dock['daemon_strikes'] ?? 0), 'daemon_restart'=> ($dock['daemon_restarted_flag'] ?? 'false') === 'true', 'ctr_strikes' => $strikes, 'skiplist' => vv_wd_parse_skiplist($skipRaw), 'sys_strikes' => $sysStrikes, 'reboots' => vv_wd_parse_reboot_log($rebootRaw), 'restarts' => vv_wd_parse_restart_log($restartRaw), 'storage_wd' => vv_wd_parse_storage_state($storRaw) + [ 'baseline_count' => $baselineCount, 'baseline_age_sec' => $baselineAgeSec, ], 'network_wd' => vv_wd_parse_network_state($netWdRaw), ], ]; } // ── Config inventory ────────────────────────────────────────────────────────── function vv_wd_node_config(string $slot, string $raw, string $masterRaw): array { $id = strtoupper($slot); return [ 'monitored' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINERS"), 'urls' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINER_URLS"), 'required' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_REQUIRED_CONTAINERS"), 'ignore' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_SCAN_IGNORE"), 'pause_list' => vv_wd_bash_array($raw, "{$id}_RW_PAUSE_CONTAINERS"), 'stop_list' => vv_wd_bash_array($raw, "{$id}_RW_STOP_CONTAINERS"), 'critical' => vv_wd_bash_array($masterRaw, "RW_CRITICAL_CONTAINERS"), // Network watchdog — host-specific 'ddns_domain' => vv_wd_scalar($raw, "{$id}_NETWORK_WATCHDOG_DDNS_DOMAIN"), 'ddns_container' => vv_wd_scalar($raw, "{$id}_NETWORK_WATCHDOG_DDNS_CONTAINER"), 'npm_url' => vv_wd_scalar($raw, "{$id}_NETWORK_WATCHDOG_NPM_URL"), // Storage watchdog — host-specific appdata suppress ceilings 'appdata_sizes' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_APPDATA_SIZES"), ]; } // ── Entry point ─────────────────────────────────────────────────────────────── function vv_wd_all(): array { $currentHost = vv_detect_host(); $tsPeers = vv_pt_ts_peers(); $masterRaw = vv_read_conf_raw('master.conf'); $restartLog = DB_DIR . '/container_restart_history.db'; // Config thresholds from master.conf $cfg = [ 'rw_soft_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_SOFT_GB') ?: 12), 'rw_medium_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_MEDIUM_GB') ?: 8), 'rw_hard_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_HARD_GB') ?: 6), 'rw_recover_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_RECOVER_GB') ?: 20), 'rw_load_soft' => (float)(vv_wd_scalar($masterRaw, 'RW_LOAD_SOFT_MULTIPLIER') ?: 2.0), 'rw_load_med' => (float)(vv_wd_scalar($masterRaw, 'RW_LOAD_MEDIUM_MULTIPLIER') ?: 3.0), 'sys_mem_gb' => (float)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_MEM_GB') ?: 4), 'sys_strikes' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_STRIKE_LIMIT') ?: 2), 'reboot_limit' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_REBOOT_LIMIT') ?: 3), 'reboot_window' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_REBOOT_WINDOW_HRS') ?: 12), 'restart_limit' => (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_CONTAINER_RESTART_LIMIT') ?: 3), 'startup_grace' => (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_STARTUP_GRACE') ?: 600), 'soft_mem_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_MEM_THRESHOLD') ?: 80), 'soft_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_CPU_THRESHOLD') ?: 80), 'hard_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'HARD_CPU_THRESHOLD') ?: 85), 'cpu_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'CPU_FAIL_LIMIT') ?: 2), 'resp_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'RESP_FAIL_LIMIT') ?: 2), // Storage watchdog 'growth_gb' => (float)(vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_GROWTH_GB') ?: 2), 'log_max_gb' => (float)(vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_LOG_MAX_GB') ?: 2), 'stor_strike_lim'=> (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_STRIKE_LIMIT') ?: 3), 'truncate_logs' => vv_wd_scalar($masterRaw, 'WATCHDOG_APPDATA_TRUNCATE_LOGS') === 'true', // Network watchdog 'net_wd_enabled' => vv_wd_scalar($masterRaw, 'NETWORK_WATCHDOG_ENABLED') !== 'false', 'npm_strike_lim' => (int)(vv_wd_scalar($masterRaw, 'NETWORK_WATCHDOG_NPM_STRIKE_LIMIT') ?: 2), 'ts_check' => vv_wd_scalar($masterRaw, 'NETWORK_WATCHDOG_CHECK_TAILSCALE') !== 'false', // Conf cache watchdog. It has no state of its own to report because it writes none until // it runs, and it does not run: conf_cache_watchdog.sh:145-146 exits on either of these // being off. Both gates ship so the page can say "off, and here is which switch", rather // than omit a member of SYSTEM_WATCHDOG_SCRIPTS entirely and let its absence read as fine. 'fallback_on' => vv_wd_scalar($masterRaw, 'FALLBACK_ENABLED') === 'true', 'conf_sync_on' => vv_wd_scalar($masterRaw, 'CONF_SYNC_ENABLED') !== 'false', ]; // SSH key from current host conf $myId = strtoupper($currentHost); $myRaw = vv_read_conf_raw($currentHost . '.conf'); $mySshKey = vv_wd_scalar($myRaw, $myId . '_SSH_KEY'); // Known hosts preg_match_all('/^\s*(HOST(\d+))(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $masterRaw, $m); $hosts = []; foreach ($m[1] as $i => $key) { $hosts['host' . $m[2][$i]] = trim($m[3][$i]); } ksort($hosts); if (!$hosts) $hosts = ['host1' => 'HOST1']; $nodes = []; foreach ($hosts as $slot => $hostname) { $isMe = ($slot === $currentHost || $currentHost === 'unknown'); $tsLabel = strtolower($hostname); $ts = $tsPeers[$tsLabel] ?? ['online' => null, 'active' => false, 'ip' => null]; $ip = $ts['ip'] ?? null; $raw = vv_read_conf_raw($slot . '.conf'); $remoteApiKey = vv_wd_scalar($raw, strtoupper($slot) . '_UNRAID_API_KEY'); if ($isMe) { $system = vv_wd_local_system(); $states = vv_wd_local_states($restartLog); } elseif ($ip && $mySshKey && $ts['online']) { $remote = vv_wd_remote_data($ip, $mySshKey, $restartLog); $system = $remote['system'] ?? null; $states = $remote['states'] ?? null; } elseif ($remoteApiKey) { $remoteStats = vv_remote_hosts_stats(); $rs = $remoteStats[strtoupper($slot)] ?? null; if ($rs && ($rs['available'] ?? false)) { $totalGb = (float)($rs['mem_total_gb'] ?? 0); $usedPct = (float)($rs['mem_used_pct'] ?? 0) / 100; $totalB = (int)($totalGb * 1073741824); $availB = (int)($totalB * (1 - $usedPct)); $system = [ 'mem_total' => $totalB, 'mem_avail' => $availB, 'load1' => 0.0, 'cores' => (int)($rs['cpu_threads'] ?? 0), 'uptime' => (int)($rs['uptime_sec'] ?? 0), 'daemon_ok' => null, 'oom_count' => 0, 'api_only' => true, ]; } else { $system = null; } $states = null; } else { $system = null; $states = null; } $nodes[] = [ 'slot' => $slot, 'id' => strtoupper($slot), 'hostname' => $hostname, 'is_me' => $isMe, 'ts_online' => $ts['online'], 'system' => $system, 'states' => $states, 'config' => vv_wd_node_config($slot, $raw, $masterRaw), ]; } return [ 'ts' => time(), 'cfg' => $cfg, 'nodes' => $nodes, ]; }