From da15cc75313679593f80d358cc0e112677bf05f4 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sat, 22 Aug 2026 00:44:35 -0400 Subject: [PATCH] Rebuild the Fallback tab around what the daemon is actually doing, not what its last state file said --- Plugin/unraid/api/fallback_control.php | 128 ++++++++++ Plugin/unraid/include/fallback.php | 94 +++++++ Plugin/unraid/pages/fallback.php | 332 ++++++++++++++++++++++--- 3 files changed, 518 insertions(+), 36 deletions(-) create mode 100644 Plugin/unraid/api/fallback_control.php diff --git a/Plugin/unraid/api/fallback_control.php b/Plugin/unraid/api/fallback_control.php new file mode 100644 index 0000000..5325d7e --- /dev/null +++ b/Plugin/unraid/api/fallback_control.php @@ -0,0 +1,128 @@ + +// +// RESPONSE +// {"ok":true,"output":string} action ran; output is the script's own report +// {"ok":false,"error":string} bad method, unknown action, or unresolvable host +// +// DEPENDS ON +// include/fallback.php vv_pt_peer_lookup(), vv_pt_ts_peers(), vv_pt_ssh() +// Fallback/fallback.sh --stop +// Fallback/fallback_test.sh --stop +// ═══════════════════════════════════════════════════════════════════════════════════════════════ +require_once dirname(__DIR__) . '/include/fallback.php'; + +header('Content-Type: application/json'); + +if ($_SERVER['REQUEST_METHOD'] !== 'POST') { + echo json_encode(['ok' => false, 'error' => 'POST only']); + exit; +} + +$action = (string)($_POST['action'] ?? ''); +$slot = strtolower((string)($_POST['host'] ?? '')); + +$allowed = ['stop', 'stop_test', 'start_dry', 'clear_lock']; +if (!in_array($action, $allowed, true)) { + echo json_encode(['ok' => false, 'error' => 'Unknown action']); + exit; +} + +$hosts = vv_fb_known_hosts(); +if (!isset($hosts[$slot])) { + echo json_encode(['ok' => false, 'error' => 'Unknown host']); + exit; +} + +$isMe = ($slot === vv_detect_host()); +$scripts = rtrim(SCRIPTS_DIR, '/'); + +// The command, as the script that owns the operation would be invoked by hand. +$cmds = [ + 'stop' => 'bash ' . escapeshellarg("$scripts/Fallback/fallback.sh") . ' --stop 2>&1', + 'stop_test' => 'bash ' . escapeshellarg("$scripts/Fallback/fallback_test.sh") . ' --stop 2>&1', + // setsid so it outlives this request; own log so the page can show what the preview said. + 'start_dry' => 'setsid bash ' . escapeshellarg("$scripts/Fallback/fallback.sh") + . ' --dry-run --log > /tmp/varaverk/fallback_dryrun.log 2>&1 < /dev/null & echo started', + 'clear_lock' => 'rm -f /tmp/unraid_locks/fallback.lock /tmp/unraid_locks/fallback_test.lock && echo cleared', +]; + +if ($isMe) { + @mkdir('/tmp/varaverk', 0755, true); + $out = (string)shell_exec($cmds[$action]); + echo json_encode(['ok' => true, 'output' => trim($out)]); + exit; +} + +// Remote: same command, same script, over the SSH this file's neighbours already use. +$tsPeers = vv_pt_ts_peers(); +$ts = vv_pt_peer_lookup($tsPeers, $hosts[$slot]); +$ip = $ts['ip'] ?? null; + +$myId = strtoupper(vv_detect_host()); +$sshKey = vv_fb_scalar(vv_read_conf_raw(vv_detect_host() . '.conf'), $myId . '_SSH_KEY'); + +if (!$ip || !$sshKey) { + echo json_encode(['ok' => false, 'error' => 'Partner not resolvable — no Tailscale IP or no SSH key']); + exit; +} + +// The remote's SCRIPTS_DIR is not this host's: appdata mode on one side and flash on the other +// is the normal case on this mesh, so ask the partner where it keeps them. +$remoteDir = trim((string)vv_pt_ssh($ip, $sshKey, + 'sed -n \'s/^SCRIPTS_DIR="\(.*\)"$/\1/p\' /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null')); +if ($remoteDir === '') $remoteDir = '/boot/config/plugins/varaverk'; + +$remoteCmds = [ + 'stop' => "bash '$remoteDir/Fallback/fallback.sh' --stop 2>&1", + 'stop_test' => "bash '$remoteDir/Fallback/fallback_test.sh' --stop 2>&1", + 'start_dry' => "mkdir -p /tmp/varaverk; setsid bash '$remoteDir/Fallback/fallback.sh'" + . " --dry-run --log > /tmp/varaverk/fallback_dryrun.log 2>&1 < /dev/null & echo started", + 'clear_lock' => 'rm -f /tmp/unraid_locks/fallback.lock /tmp/unraid_locks/fallback_test.lock && echo cleared', +]; + +$out = vv_pt_ssh($ip, $sshKey, $remoteCmds[$action]); +echo json_encode(['ok' => true, 'output' => trim((string)$out)]); diff --git a/Plugin/unraid/include/fallback.php b/Plugin/unraid/include/fallback.php index 4d23936..5e293f2 100644 --- a/Plugin/unraid/include/fallback.php +++ b/Plugin/unraid/include/fallback.php @@ -115,6 +115,63 @@ function vv_fb_remote_running(string $ip, string $sshKey): array { return array_values(array_filter(explode("\n", trim($out)))); } + +// ── Daemon / process state ──────────────────────────────────────────────────── +// +// The page had no way to say whether fallback.sh was running at all, which is the first thing +// anyone looking at this tab wants to know — every state below is written BY that daemon, so a +// stale NORMAL from a process that died days ago read exactly like a healthy one. +// +// Mode matters as much as liveness. A --dry-run instance takes the same `fallback` lock as the +// real daemon, so the lock alone cannot tell them apart; the cmdline can, and the per-PID +// dry-run state copy is a second confirmation. +function vv_fb_proc(string $lockName): array { + $lockFile = '/tmp/unraid_locks/' . $lockName . '.lock'; + $out = ['running' => false, 'pid' => null, 'mode' => null, 'since' => null, 'stale_lock' => false]; + + if (!is_file($lockFile)) return $out; + + $pid = (int)strtok((string)@file_get_contents($lockFile), ':'); + // A lock whose PID is gone is not "running" — it is residue from a SIGKILL or a power cut, + // and saying so is the difference between "stop it" and "clear it". + if ($pid <= 0 || !is_dir("/proc/$pid")) { + $out['stale_lock'] = true; + $out['pid'] = $pid ?: null; + return $out; + } + + $cmd = (string)@file_get_contents("/proc/$pid/cmdline"); + $args = explode("\0", $cmd); + $out['running'] = true; + $out['pid'] = $pid; + $out['mode'] = in_array('--dry-run', $args, true) || in_array('-n', $args, true) ? 'dry-run' : 'live'; + $st = @stat("/proc/$pid"); + if ($st) $out['since'] = (int)$st['mtime']; + return $out; +} + +// Per-host daemon state. Local reads /proc directly; a partner is asked over the same SSH the +// rest of this file already uses, in one call rather than three. +function vv_fb_remote_proc(string $ip, string $sshKey): array { + $cmd = 'for L in fallback fallback_test; do F=/tmp/unraid_locks/$L.lock; ' + . 'if [ -f "$F" ]; then P=$(cut -d: -f1 "$F"); ' + . 'if [ -d "/proc/$P" ]; then M=live; tr "\\0" " " < /proc/$P/cmdline | grep -q -- "--dry-run" && M=dry-run; ' + . 'echo "$L:running:$P:$M"; else echo "$L:stale:$P:"; fi; else echo "$L:none::"; fi; done'; + $out = vv_pt_ssh($ip, $sshKey, $cmd); + $res = ['fallback' => ['running' => false, 'pid' => null, 'mode' => null, 'stale_lock' => false], + 'fallback_test' => ['running' => false, 'pid' => null, 'mode' => null, 'stale_lock' => false]]; + foreach (explode("\n", trim((string)$out)) as $line) { + $p = explode(':', trim($line)); + if (count($p) < 4 || !isset($res[$p[0]])) continue; + if ($p[1] === 'running') { + $res[$p[0]] = ['running' => true, 'pid' => (int)$p[2], 'mode' => $p[3] ?: 'live', 'stale_lock' => false]; + } elseif ($p[1] === 'stale') { + $res[$p[0]]['stale_lock'] = true; + $res[$p[0]]['pid'] = (int)$p[2] ?: null; + } + } + return $res; +} // ── Covers — what a node runs for the other when it's down ─────────────────── function vv_fb_covers(string $covering, string $remote, string $coveringRaw, string $remoteRaw): array { @@ -207,14 +264,51 @@ function vv_fb_all(): array { break; // 2-node only } + // Daemon liveness per node. Everything in $state was written by this process — without + // it a NORMAL left behind by a daemon that died days ago is indistinguishable from a + // NORMAL being refreshed every 30 seconds. + if ($isMe) { + $proc = vv_fb_proc('fallback'); + $procTest = vv_fb_proc('fallback_test'); + } elseif ($ip && $mySshKey && $ts['online']) { + $rp = vv_fb_remote_proc($ip, $mySshKey); + $proc = $rp['fallback']; + $procTest = $rp['fallback_test']; + } else { + $proc = ['running' => null, 'pid' => null, 'mode' => null, 'stale_lock' => false]; + $procTest = ['running' => null, 'pid' => null, 'mode' => null, 'stale_lock' => false]; + } + + // How fresh the state actually is. The daemon rewrites its file every check interval, so + // an age far past that interval means it is wedged even while the process still exists. + $stateAge = null; + if ($isMe) { + $sp = STATE_DIR . '/fallback_state.db'; + if (is_file($sp)) $stateAge = time() - (int)@filemtime($sp); + } + + // "Reachable" is three separate facts and one boolean hid which had failed. + $reach = [ + 'tailscale' => $isMe ? true : ($ts['online'] === true), + 'ip' => $isMe ? null : $ip, + 'ssh' => $isMe ? true : ($running !== [] || ($proc['running'] !== null)), + 'state_file' => ($state['state'] ?? 'UNKNOWN') !== 'UNKNOWN', + ]; + $nodes[] = [ 'slot' => $slot, 'id' => strtoupper($slot), 'hostname' => $hostname, 'is_me' => $isMe, 'ts_online' => $ts['online'], + 'ts_ip' => $ip, 'state' => $state, + 'state_age' => $stateAge, 'running' => $running, + 'running_count' => count($running), + 'proc' => $proc, + 'proc_test' => $procTest, + 'reach' => $reach, 'covers' => $covers, ]; } diff --git a/Plugin/unraid/pages/fallback.php b/Plugin/unraid/pages/fallback.php index 496f47c..16534f0 100644 --- a/Plugin/unraid/pages/fallback.php +++ b/Plugin/unraid/pages/fallback.php @@ -16,22 +16,81 @@ // outage, when stale numbers are actively misleading. // // OPERATIONAL SAFEGUARDS -// Read-only. The page cannot trigger a failover, force a handback, or start a covered -// container. Fallback is driven by fallback.sh reacting to real reachability, and a manual -// override from a browser is exactly the wrong way to enter that state. +// Cannot cause a failover. The page stops daemons, starts a PREVIEW, and edits conf — it has +// no control that enters FALLBACK or forces a handback. Fallback is driven by fallback.sh +// reacting to real reachability, and a manual override from a browser is exactly the wrong +// way to enter that state. The one start control is --dry-run, which changes nothing. // // A missing state file renders as unknown, never as NORMAL — claiming healthy for a // fallback process that is not running would be the worst possible error on this page. // +// State age is shown next to state, always. Every value in the state file was written by a +// daemon that may not be running: this host displayed NORMAL from a file five days stale +// with no process alive, and nothing on the page said so. +// +// Stopping the test is a different button from stopping the daemon, deliberately. The test +// holds an iptables rule that only its own EXIT trap removes, so the two cannot share a +// control that might escalate to SIGKILL. +// // RENDERS -// Per-node state, tier activation and delays, handback strikes, covered container status +// Per-node state and freshness, daemon liveness and mode, reachability legs, tier activation +// and delays, handback strikes, covered container status, quick settings, assistant // // DEPENDS ON -// api/fallback.php polled every 30s → include/fallback.php -// api/confform.php inline conf edits → include/confform.php +// api/fallback.php polled every 30s → include/fallback.php +// api/fallback_control.php start/stop actions → Fallback/fallback*.sh +// api/confform.php inline conf edits → include/confform.php require_once dirname(__DIR__) . '/include/confui.php'; +require_once dirname(__DIR__) . '/include/ai_chat.php'; + +// Same shared surface as the Partnership and Monitor tabs, gated the same way. Fallback +// questions — why a tier has not fired, what the mirror would actually start, whether a stale +// state file matters — are asked while looking at this page. +if (vv_ai_ui_on()) vv_ai_chat_assets(); ?>