_TIER. // Named for the host being covered, not the host doing the covering — see the fallback // section of the top-level README for why that reads backwards at first. // // OPERATIONAL SAFEGUARDS // A missing state file parses as empty, not as NORMAL. // vv_fb_local_state() hands an empty string to the parser when the file is absent, so // the page shows unknown rather than asserting everything is fine. Reporting a healthy // state for a fallback process that is not running would be the worst possible lie on // this page. // // An unreachable partner degrades to what is locally known. // Remote SSH failures return empty rather than propagating an error, so one dark node // cannot blank the whole page — which is precisely the situation this page exists for. // // Read-only over SSH. // The only remote commands issued are a state-file read and `docker ps`. Nothing here // starts or stops a container on either side. // // EXPORTS // State vv_fb_local_state(), vv_fb_remote_state(), vv_fb_parse_state() // Containers vv_fb_local_running(), vv_fb_remote_running(), vv_fb_covers() // Assembly vv_fb_known_hosts(), vv_fb_all() // Parsing vv_fb_bash_array(), vv_fb_scalar() // // CONFIGURATION // STATE_DIR fallback_state.db — written by Fallback/fallback.sh // FALLBACK__TIER1..4 containers covered per tier, keyed by the covered host // HOST*_SSH_KEY used to read partner state // ═══════════════════════════════════════════════════════════════════════════════════════════════ require_once __DIR__ . '/config.php'; require_once __DIR__ . '/partnership.php'; // vv_pt_ssh(), vv_pt_ts_peers() // ── Conf parsers ────────────────────────────────────────────────────────────── function vv_fb_bash_array(string $raw, string $varname): array { return vv_parse_bash_array($raw, $varname); } function vv_fb_scalar(string $raw, string $varname): string { return vv_parse_conf_scalar($raw, $varname); } // ── State file ──────────────────────────────────────────────────────────────── function vv_fb_parse_state(string $text): array { $out = [ 'state' => 'UNKNOWN', 'fallback_start' => 0, 'handback_strikes' => 0, 'tier2_started' => false, 'tier3_started' => false, 'tier4_started' => false, 'partnership_suspended' => false, 'partner_lost_at' => 0, ]; foreach (explode("\n", $text) as $line) { $line = trim($line); if (!$line || !str_contains($line, '=')) continue; [$k, $v] = array_pad(explode('=', $line, 2), 2, ''); $k = trim($k); $v = trim($v, '"\''); switch ($k) { case 'state': $out['state'] = $v; break; case 'fallback_start': $out['fallback_start'] = (int)$v; break; case 'handback_strikes': $out['handback_strikes'] = (int)$v; break; case 'tier2_started': $out['tier2_started'] = $v === 'true'; break; case 'tier3_started': $out['tier3_started'] = $v === 'true'; break; case 'tier4_started': $out['tier4_started'] = $v === 'true'; break; case 'partnership_suspended': $out['partnership_suspended'] = $v === 'true'; break; case 'partner_lost_at': $out['partner_lost_at'] = (int)$v; break; } } return $out; } function vv_fb_local_state(): array { $path = STATE_DIR . '/fallback_state.db'; return vv_fb_parse_state(file_exists($path) ? file_get_contents($path) : ''); } function vv_fb_remote_state(string $ip, string $sshKey): array { $out = vv_pt_ssh($ip, $sshKey, vv_remote_state_cmd('fallback_state.db')); return vv_fb_parse_state($out); } // ── Running containers ──────────────────────────────────────────────────────── function vv_fb_local_running(): array { $out = shell_exec("docker ps --format '{{.Names}}' 2>/dev/null") ?: ''; return array_values(array_filter(explode("\n", trim($out)))); } function vv_fb_remote_running(string $ip, string $sshKey): array { $out = vv_pt_ssh($ip, $sshKey, "docker ps --format '{{.Names}}' 2>/dev/null"); return array_values(array_filter(explode("\n", trim($out)))); } // ── Daemon / process state ──────────────────────────────────────────────────── // // The page had no way to say whether fallback.sh was running at all, which is the first thing // anyone looking at this tab wants to know — every state below is written BY that daemon, so a // stale NORMAL from a process that died days ago read exactly like a healthy one. // // Mode matters as much as liveness. A --dry-run instance takes the same `fallback` lock as the // real daemon, so the lock alone cannot tell them apart; the cmdline can, and the per-PID // dry-run state copy is a second confirmation. function vv_fb_proc(string $lockName): array { $lockFile = '/tmp/unraid_locks/' . $lockName . '.lock'; $out = ['running' => false, 'pid' => null, 'mode' => null, 'since' => null, 'stale_lock' => false]; if (!is_file($lockFile)) return $out; $pid = (int)strtok((string)@file_get_contents($lockFile), ':'); // A lock whose PID is gone is not "running" — it is residue from a SIGKILL or a power cut, // and saying so is the difference between "stop it" and "clear it". if ($pid <= 0 || !is_dir("/proc/$pid")) { $out['stale_lock'] = true; $out['pid'] = $pid ?: null; return $out; } $cmd = (string)@file_get_contents("/proc/$pid/cmdline"); $args = explode("\0", $cmd); $out['running'] = true; $out['pid'] = $pid; $out['mode'] = in_array('--dry-run', $args, true) || in_array('-n', $args, true) ? 'dry-run' : 'live'; $st = @stat("/proc/$pid"); if ($st) $out['since'] = (int)$st['mtime']; return $out; } // Per-host daemon state. Local reads /proc directly; a partner is asked over the same SSH the // rest of this file already uses, in one call rather than three. function vv_fb_remote_proc(string $ip, string $sshKey): array { $cmd = 'for L in fallback fallback_test; do F=/tmp/unraid_locks/$L.lock; ' . 'if [ -f "$F" ]; then P=$(cut -d: -f1 "$F"); ' . 'if [ -d "/proc/$P" ]; then M=live; tr "\\0" " " < /proc/$P/cmdline | grep -q -- "--dry-run" && M=dry-run; ' . 'echo "$L:running:$P:$M"; else echo "$L:stale:$P:"; fi; else echo "$L:none::"; fi; done'; $out = vv_pt_ssh($ip, $sshKey, $cmd); $res = ['fallback' => ['running' => false, 'pid' => null, 'mode' => null, 'stale_lock' => false], 'fallback_test' => ['running' => false, 'pid' => null, 'mode' => null, 'stale_lock' => false]]; foreach (explode("\n", trim((string)$out)) as $line) { $p = explode(':', trim($line)); if (count($p) < 4 || !isset($res[$p[0]])) continue; if ($p[1] === 'running') { $res[$p[0]] = ['running' => true, 'pid' => (int)$p[2], 'mode' => $p[3] ?: 'live', 'stale_lock' => false]; } elseif ($p[1] === 'stale') { $res[$p[0]]['stale_lock'] = true; $res[$p[0]]['pid'] = (int)$p[2] ?: null; } } return $res; } // A running dry run keeps its own state file — VV_CACHE_ROOT/fallback_state.dryrun. — and // refreshes it every check interval, exactly as the live daemon would. Reading the LIVE file // while a preview is running is how a healthy dry run came to render as UNKNOWN on a host that // has simply never run fallback for real: the answer existed, in a file next to the one being // read. The preview is shown as the preview, never merged into the live state. function vv_fb_dryrun_state(int $pid): array { $p = (VV_CACHE_ROOT ?: '/tmp/varaverk') . '/fallback_state.dryrun.' . $pid; if (!is_file($p)) return ['state' => null, 'age' => null]; $s = vv_fb_parse_state((string)@file_get_contents($p)); $s['age'] = time() - (int)@filemtime($p); return $s; } function vv_fb_remote_dryrun_state(string $ip, string $sshKey, int $pid): array { $f = '/tmp/varaverk/fallback_state.dryrun.' . $pid; $out = vv_pt_ssh($ip, $sshKey, "[ -f '$f' ] && { echo \"__age=\$(( \$(date +%s) - \$(stat -c %Y '$f') ))\"; cat '$f'; }"); if (trim((string)$out) === '') return ['state' => null, 'age' => null]; $age = null; if (preg_match('/^__age=(\d+)/m', (string)$out, $m)) $age = (int)$m[1]; $s = vv_fb_parse_state((string)$out); $s['age'] = $age; return $s; } // ── Covers — what a node runs for the other when it's down ─────────────────── function vv_fb_covers(string $covering, string $remote, string $coveringRaw, string $remoteRaw): array { $ru = strtoupper($remote); // HOST2 — covered host owns its own tier lists $tiers = []; for ($t = 1; $t <= 4; $t++) { $tiers["tier$t"] = vv_fb_bash_array($remoteRaw, "FALLBACK_{$ru}_TIER{$t}"); } // Delays: how long the remote (covered) host must be down before each tier fires. // Stored in the *remote* host's conf as REMOTE_TIER*_DELAY. $tiers['delays'] = [ 'tier2' => (int)(vv_fb_scalar($remoteRaw, "{$ru}_TIER2_DELAY") ?: 240), 'tier3' => (int)(vv_fb_scalar($remoteRaw, "{$ru}_TIER3_DELAY") ?: 720), 'tier4' => (int)(vv_fb_scalar($remoteRaw, "{$ru}_TIER4_DELAY") ?: 1440), ]; return $tiers; } // ── Known hosts ─────────────────────────────────────────────────────────────── function vv_fb_known_hosts(): array { return vv_known_hosts(); } // ── Main data builder ───────────────────────────────────────────────────────── function vv_fb_all(): array { $currentHost = vv_detect_host(); $hosts = vv_fb_known_hosts(); $tsPeers = vv_pt_ts_peers(); $masterRaw = vv_read_conf_raw('master.conf'); $handbackReq = (int)(vv_fb_scalar($masterRaw, 'FALLBACK_HANDBACK_STRIKES') ?: 3); $fbEnabled = vv_fb_scalar($masterRaw, 'FALLBACK_ENABLED') === 'true'; $ptEnabled = vv_fb_scalar($masterRaw, 'PARTNERSHIP_ENABLED') === 'true'; $rsyncEnabled = vv_fb_scalar($masterRaw, 'FALLBACK_RSYNC_ENABLED') !== 'false'; $checkInterval = (int)(vv_fb_scalar($masterRaw, 'FALLBACK_CHECK_INTERVAL') ?: 30); $suspendAfter = (int)(vv_fb_scalar($masterRaw, 'FALLBACK_PARTNERSHIP_SUSPEND_AFTER') ?: 120); // Read all host conf raws upfront $raws = []; foreach (array_keys($hosts) as $slot) { $raws[$slot] = vv_read_conf_raw($slot . '.conf'); } // SSH key — from local host conf $myId = strtoupper($currentHost); $myRaw = $raws[$currentHost] ?? ''; $mySshKey = vv_fb_scalar($myRaw, $myId . '_SSH_KEY'); $nodes = []; foreach ($hosts as $slot => $hostname) { $isMe = ($slot === $currentHost || $currentHost === 'unknown'); // Exact-key only, until now — see vv_pt_peer_lookup(). This mesh's conf name and tailnet // name differ by one character, so HOST2 missed every lookup and the page rendered a // partner that was up the whole time as state=UNREACHABLE. $ts = vv_pt_peer_lookup($tsPeers, $hostname); $ip = $ts['ip'] ?? null; // State if ($isMe) { $state = vv_fb_local_state(); } elseif ($ip && $mySshKey) { $state = vv_fb_remote_state($ip, $mySshKey); } else { $state = vv_fb_parse_state(''); $state['state'] = $ts['online'] === false ? 'OFFLINE' : 'UNREACHABLE'; } // Running containers if ($isMe) { $running = vv_fb_local_running(); } elseif ($ip && $mySshKey && $ts['online']) { $running = vv_fb_remote_running($ip, $mySshKey); } else { $running = []; } // Covers: for a 2-node setup, each covers the other // For N nodes this would need a different approach — for now, assume 2-node $covers = null; foreach ($hosts as $otherSlot => $otherHostname) { if ($otherSlot === $slot) continue; $coveringRaw = $raws[$slot] ?? ''; $remoteRaw = $raws[$otherSlot] ?? ''; $covers = [ 'slot' => $otherSlot, 'id' => strtoupper($otherSlot), 'hostname' => $otherHostname, ] + vv_fb_covers($slot, $otherSlot, $coveringRaw, $remoteRaw); break; // 2-node only } // Daemon liveness per node. Everything in $state was written by this process — without // it a NORMAL left behind by a daemon that died days ago is indistinguishable from a // NORMAL being refreshed every 30 seconds. if ($isMe) { $proc = vv_fb_proc('fallback'); $procTest = vv_fb_proc('fallback_test'); } elseif ($ip && $mySshKey && $ts['online']) { $rp = vv_fb_remote_proc($ip, $mySshKey); $proc = $rp['fallback']; $procTest = $rp['fallback_test']; } else { $proc = ['running' => null, 'pid' => null, 'mode' => null, 'stale_lock' => false]; $procTest = ['running' => null, 'pid' => null, 'mode' => null, 'stale_lock' => false]; } // When a preview is running, read what IT is deciding — kept beside the live file and // refreshed on the same interval. Reported separately so the live state is never // overwritten by a preview's opinion. $preview = ['state' => null, 'age' => null]; if (($proc['mode'] ?? '') === 'dry-run' && !empty($proc['pid'])) { $preview = $isMe ? vv_fb_dryrun_state((int)$proc['pid']) : vv_fb_remote_dryrun_state($ip, $mySshKey, (int)$proc['pid']); } // How fresh the state actually is. The daemon rewrites its file every check interval, so // an age far past that interval means it is wedged even while the process still exists. $stateAge = null; if ($isMe) { $sp = STATE_DIR . '/fallback_state.db'; if (is_file($sp)) $stateAge = time() - (int)@filemtime($sp); } // "Reachable" is three separate facts and one boolean hid which had failed. $reach = [ 'tailscale' => $isMe ? true : ($ts['online'] === true), 'ip' => $isMe ? null : $ip, 'ssh' => $isMe ? true : ($running !== [] || ($proc['running'] !== null)), 'state_file' => ($state['state'] ?? 'UNKNOWN') !== 'UNKNOWN', ]; $nodes[] = [ 'slot' => $slot, 'id' => strtoupper($slot), 'hostname' => $hostname, 'is_me' => $isMe, 'ts_online' => $ts['online'], 'ts_ip' => $ip, 'state' => $state, 'state_age' => $stateAge, 'preview' => $preview, 'running' => $running, 'running_count' => count($running), 'proc' => $proc, 'proc_test' => $procTest, 'reach' => $reach, 'covers' => $covers, ]; } return [ 'ts' => time(), 'fb_enabled' => $fbEnabled, 'partnership_enabled' => $ptEnabled, 'fb_rsync_enabled' => $rsyncEnabled, 'handback_req' => $handbackReq, 'check_interval' => $checkInterval, 'suspend_after' => $suspendAfter, 'nodes' => $nodes, ]; }