diff --git a/Fallback/coverage_deploy.sh b/Fallback/coverage_deploy.sh new file mode 100755 index 0000000..418bdd8 --- /dev/null +++ b/Fallback/coverage_deploy.sh @@ -0,0 +1,315 @@ +#!/bin/bash +# ══════════════════════════════════════════════════════════════════════════════════════════════ +# PURPOSE +# Put the containers this host has marked for fallback coverage onto the partner, so that the +# partner can actually start them during an outage — and take them off again on request. +# +# OPERATIONAL MODEL +# fallback.sh covers a host by running `docker start ` on the partner. It never creates +# anything. So a name in FALLBACK__TIER* is a promise that only holds if the partner already +# has that container built. Measured 2026-08-23: all 12 of HOST1's covered containers were absent +# from HOST2, meaning every tier would have failed on the first real outage while the UI showed +# coverage as configured. This script is what closes that gap. +# +# Push and remove are separate, deliberate actions, never a side effect of saving the tier list. +# Editing coverage is a cheap config write; deploying a dozen containers onto another machine is +# not, and the two should not share a button. +# +# DESIGN PRINCIPLES +# Deployed, then verified STOPPED. +# A container built here and left running on the partner would be a second live instance of +# NextCloud, Gitea or PostgreSQL_Immich against the same data while this host is healthy. +# That is the danger_rsync_live_database_appdata failure with worse odds. Every deploy is +# followed by a stop and a re-inspect, and a container that will not stay stopped is an +# error, not a warning. +# +# Remove takes the container AND its appdata. +# Operator decision 2026-08-23: the button is explicit, so a removal should leave nothing +# behind to reason about later. The risk it accepts is narrow and worth naming — if the +# partner ever covered for us, ITS appdata is the newer copy and is what a handback rsyncs +# home. The NORMAL-state gate below closes the live-failover window; what it cannot see is +# a handback that partially failed and then returned to NORMAL, so the UI says so before +# asking. +# +# Two guards on the deletion itself: only paths under /mnt/*/appdata* are ever touched, and +# a bind of the appdata ROOT is refused outright — a container mounting /mnt/user/appdata +# would otherwise turn one removal into wiping every application on the partner. +# +# Refuses to run unless fallback state is NORMAL. +# Pushing or removing containers mid-outage edits the thing currently keeping services up. +# +# Coverage names are resolved to templates by , not by filename. +# my-Foo.xml routinely holds a container called something else. Matching on the filename +# silently pushes the wrong template, or nothing at all. +# +# USAGE +# coverage_deploy.sh --push deploy every covered container onto the partner (stopped) +# coverage_deploy.sh --remove stop, remove, and delete the pushed template on the partner +# coverage_deploy.sh --status report, per covered container, whether it exists there +# any mode supports --dry-run +# +# DEPENDS ON +# Plugin//Partnership/containers.sh deploy_container_from_xml(), GPU transform +# FALLBACK__TIER1-4 the coverage list this acts on +# ══════════════════════════════════════════════════════════════════════════════════════════════ + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../load_config.sh" +source "$SCRIPT_DIR/../Plugin/$PLATFORM/Partnership/containers.sh" + +SSH_TIMEOUT="${SSH_TIMEOUT:-15}" +MODE="" +DRY_RUN="${DRY_RUN:-false}" + +for arg in "$@"; do + case "$arg" in + --push) MODE="push" ;; + --remove) MODE="remove" ;; + --status) MODE="status" ;; + --dry-run) DRY_RUN=true ;; + esac +done + +if [[ -z "$MODE" ]]; then + error "No mode given — use --push, --remove or --status" + exit 1 +fi + +detect_hosts + +if [[ -z "$REMOTE_ID" || "$REMOTE_SERVER_NAME" == "unknown" ]]; then + error "No partner configured — nothing to push to" + exit 1 +fi + +# ── Gate: only with fallback idle ───────────────────────────────────────────────────────────── +# Read rather than assumed. A missing state file means fallback has never run, which is idle +# enough; a file that says anything other than NORMAL means services are in motion right now. +FALLBACK_STATE_FILE="${FALLBACK_STATE_FILE:-${STATE_DIR}/fallback_state.db}" +_fb_state="NORMAL" +if [[ -f "$FALLBACK_STATE_FILE" ]]; then + _fb_state=$(grep -m1 '^state=' "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) + _fb_state="${_fb_state:-NORMAL}" +fi +if [[ "$_fb_state" != "NORMAL" && "$MODE" != "status" ]]; then + error "Fallback state is $_fb_state, not NORMAL — refusing to $MODE" + error "Changing what the partner holds while a failover is live edits the thing keeping services up." + exit 1 +fi + +# ── The coverage list ───────────────────────────────────────────────────────────────────────── +COVERED=() +for _t in 1 2 3 4; do + _var="FALLBACK_${MY_ID}_TIER${_t}[@]" + for _c in "${!_var}"; do + [[ -n "$_c" ]] && COVERED+=("$_c") + done +done + +if [[ ${#COVERED[@]} -eq 0 ]]; then + warn "No containers are covered in FALLBACK_${MY_ID}_TIER1-4 — nothing to do" + exit 0 +fi + +log "$ICON_FALLBACK Coverage: ${#COVERED[@]} container(s) for $REMOTE_SERVER_NAME to start during an outage" + +resolve_remote_ip +MIRROR="$REMOTE_SERVER_NAME" +MIRROR_IP="$REMOTE_SERVER" +_key_var="${MY_ID}_SSH_KEY" +MIRROR_SSH_KEY="${!_key_var}" + +if [[ ! -f "$MIRROR_SSH_KEY" ]]; then + error "SSH key $MIRROR_SSH_KEY not found — cannot reach $MIRROR" + exit 1 +fi + +# ── name -> template ────────────────────────────────────────────────────────────────────────── +# Matched on the element. Filenames lie often enough that trusting them would push the +# wrong container without saying so. +xml_for_container() { + local want="$1" f n + for f in "$TEMPLATES_DIR"/*.xml; do + [[ -f "$f" ]] || continue + n=$(awk 'match($0,/([^<]+)<\/Name>/,a){print a[1];exit}' "$f") + [[ "$n" == "$want" ]] && { echo "$f"; return 0; } + done + return 1 +} + +remote_has_container() { + timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \ + -o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \ + "docker inspect $(printf '%q' "$1") >/dev/null 2>&1" 2>/dev/null +} + +remote_state_of() { + timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \ + -o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \ + "docker inspect -f '{{.State.Status}}' $(printf '%q' "$1") 2>/dev/null" 2>/dev/null +} + +OK=0; FAIL=0; SKIP=0 + +case "$MODE" in + +status) + # Written as a cache as well as printed. The assistant's fallback_state block cannot afford an + # SSH round trip per container mid-question, so it reads this file and reports its AGE — a stale + # answer stated as stale is useful, stated as current it is the exact failure this feature + # exists to prevent. + _present="" _missing="" + for c in "${COVERED[@]}"; do + if remote_has_container "$c"; then + _st=$(remote_state_of "$c") + printf ' %-28s on %s (%s)\n' "$c" "$MIRROR" "$_st" + _present+="\"$c\":\"${_st:-unknown}\"," + OK=$((OK+1)) + else + printf ' %-28s MISSING on %s — docker start would fail\n' "$c" "$MIRROR" + _missing+="\"$c\"," + FAIL=$((FAIL+1)) + fi + done + log "$ICON_FALLBACK Coverage present: $OK · missing: $FAIL" + + mkdir -p "$VV_CACHE_ROOT/api" 2>/dev/null || mkdir -p /tmp/varaverk/api 2>/dev/null + _cache="${VV_CACHE_ROOT:-/tmp/varaverk}/api/fallback_presence.json" + # Written atomically — a half-written cache read mid-question would report containers as + # missing that are merely unparsed. + printf '{"present":{%s},"missing":[%s],"partner":"%s","checked":%s}\n' \ + "${_present%,}" "${_missing%,}" "$MIRROR" "$(date +%s)" > "$_cache.tmp" \ + && mv -f "$_cache.tmp" "$_cache" + + [[ "$FAIL" -gt 0 ]] && exit 2 || exit 0 + ;; + +push) + # Networks first — a container whose network is absent is created and then cannot start, + # which is the failure that read as "auth 0/8, arr 0/5" during onboarding. + _nets=() + for c in "${COVERED[@]}"; do + x=$(xml_for_container "$c") || continue + net=$(sed -n 's/.*\([^<]*\)<\/Network>.*/\1/p' "$x" 2>/dev/null | head -1) + net="${net//[[:space:]]/}" + # br* is host hardware. wg* is a WireGuard-backed bridge whose meaning does NOT travel: + # recreating it on the partner as a plain bridge yields a network that exists, starts its + # containers, and routes their traffic OUTSIDE the tunnel. ChannelTube rides wg0 here. + case "$net" in + ''|bridge|host|none|br[0-9]*) continue ;; + wg[0-9]*) + warn "$c uses $net — a WireGuard-backed network. NOT created on $MIRROR: a plain" + warn " bridge of the same name would route its traffic outside the tunnel. Build the" + warn " matching tunnel there first, or drop $c from coverage." + continue ;; + esac + _seen=false + for n in "${_nets[@]}"; do [[ "$n" == "$net" ]] && { _seen=true; break; }; done + [[ "$_seen" == false ]] && _nets+=("$net") + done + for net in "${_nets[@]}"; do + driver=$(timeout "${DOCKER_TIMEOUT:-30}" docker network inspect "$net" --format '{{.Driver}}' 2>/dev/null) + if [[ "$driver" != "bridge" ]]; then + warn "Network $net is '${driver:-absent}' here, not bridge — create it on $MIRROR by hand" + continue + fi + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would ensure network $net on $MIRROR" + continue + fi + timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \ + -o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \ + "docker network inspect $(printf '%q' "$net") >/dev/null 2>&1 \ + || docker network create --driver bridge $(printf '%q' "$net") >/dev/null" 2>/dev/null \ + && log " network $net ready on $MIRROR" \ + || warn " could not ensure network $net on $MIRROR" + done + + for c in "${COVERED[@]}"; do + x=$(xml_for_container "$c") || { + warn "$c — no template in $TEMPLATES_DIR names it; skipped" + SKIP=$((SKIP+1)); continue + } + if ! deploy_container_from_xml "$x" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then + error "$c — deploy failed" + FAIL=$((FAIL+1)); continue + fi + if [[ "$DRY_RUN" == true ]]; then OK=$((OK+1)); continue; fi + + # Deployed containers must not run here. Stop, then re-inspect — a stop that did not take + # is the one outcome that silently duplicates a live service against shared data. + timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \ + -o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \ + "docker stop $(printf '%q' "$c") >/dev/null 2>&1" 2>/dev/null + st=$(remote_state_of "$c") + if [[ "$st" == "running" ]]; then + error "$c is RUNNING on $MIRROR after deploy and would not stop — stop it there before continuing" + FAIL=$((FAIL+1)) + else + log " $c deployed and ${st:-stopped} on $MIRROR ✅" + OK=$((OK+1)) + fi + done + log "$ICON_FALLBACK Push complete — deployed $OK · failed $FAIL · skipped $SKIP" + [[ "$FAIL" -gt 0 ]] && exit 1 || exit 0 + ;; + +remove) + for c in "${COVERED[@]}"; do + if ! remote_has_container "$c"; then + log " $c not on $MIRROR — nothing to remove" + SKIP=$((SKIP+1)); continue + fi + # Binds are read BEFORE the container goes — once it is removed there is nothing left to + # enumerate, and a path list gathered afterwards would silently be empty. + _binds=$(timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \ + -o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \ + "docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' $(printf '%q' "$c") 2>/dev/null \ + | awk -F: '{print \$1}'" 2>/dev/null) + + _wipe=() + while IFS= read -r _p; do + [[ -z "$_p" ]] && continue + # Only appdata, and never an appdata root. /mnt/user/appdata as a bind would make one + # container removal delete every application on the partner. + [[ "$_p" =~ ^/mnt/[^/]+/appdata[^/]*/.+ ]] || { + [[ "$_p" =~ ^/mnt/[^/]+/appdata[^/]*/?$ ]] && \ + warn " $c binds the appdata ROOT ($_p) — refusing to delete it" + continue + } + _wipe+=("$_p") + done <<< "$_binds" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would stop and remove $c on $MIRROR" + for _p in "${_wipe[@]}"; do warn " DRY RUN — would delete appdata $_p on $MIRROR"; done + OK=$((OK+1)); continue + fi + x=$(xml_for_container "$c") && xml_name=$(basename "$x") || xml_name="" + if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \ + -o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \ + "docker stop $(printf '%q' "$c") >/dev/null 2>&1; \ + docker rm $(printf '%q' "$c") >/dev/null 2>&1; \ + ${xml_name:+rm -f ${TEMPLATES_DIR}/$(printf '%q' "$xml_name");} \ + ! docker inspect $(printf '%q' "$c") >/dev/null 2>&1" 2>/dev/null; then + log " $c removed from $MIRROR ✅" + for _p in "${_wipe[@]}"; do + if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \ + -o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \ + "rm -rf -- $(printf '%q' "$_p") && ! [ -e $(printf '%q' "$_p") ]" 2>/dev/null; then + log " appdata deleted on $MIRROR: $_p" + else + warn " could not delete appdata on $MIRROR: $_p" + FAIL=$((FAIL+1)) + fi + done + OK=$((OK+1)) + else + error "$c — removal failed or it still exists on $MIRROR" + FAIL=$((FAIL+1)) + fi + done + log "$ICON_FALLBACK Remove complete — removed $OK · failed $FAIL · skipped $SKIP" + [[ "$FAIL" -gt 0 ]] && exit 1 || exit 0 + ;; +esac diff --git a/Plugin/unraid/Tools/ai_chat_worker.php b/Plugin/unraid/Tools/ai_chat_worker.php index dabfcf2..0ba5d47 100644 --- a/Plugin/unraid/Tools/ai_chat_worker.php +++ b/Plugin/unraid/Tools/ai_chat_worker.php @@ -422,6 +422,17 @@ if ($can('system_state')) { } } +// Fallback is dormant until it isn't, so "configured" and "would work" are unrelated — this is the +// only block that reports the second. Cheap: local conf and state file plus one cached presence +// read, never a network round trip. +if ($can('fallback_state')) { + $fb = vv_ai_fallback_state(); + if ($fb !== '') { + $attached['fallback'] = 'readiness'; + $diagBlock .= $fb; + } +} + // The troubleshooting profile gets the actual tail of the one log the operator is looking at, // warnings and ordinary lines alike. The fleet-wide WARN/ERROR sweep above cannot answer "why // did this one stop" — the last line a script printed before dying is usually not labelled. diff --git a/Plugin/unraid/api/fallback_coverage.php b/Plugin/unraid/api/fallback_coverage.php index 1d07765..ee055a7 100644 --- a/Plugin/unraid/api/fallback_coverage.php +++ b/Plugin/unraid/api/fallback_coverage.php @@ -59,6 +59,7 @@ // include/config.php vv_detect_host(), vv_read_conf_raw(), vv_push_master_conf() // ═══════════════════════════════════════════════════════════════════════════════════════════════ header('Content-Type: application/json'); +require_once dirname(__DIR__) . '/include/fallback.php'; // vv_fb_proc() require_once dirname(__DIR__) . '/include/confform.php'; require_once dirname(__DIR__) . '/include/common.php'; @@ -143,6 +144,191 @@ if ($_SERVER['REQUEST_METHOD'] !== 'POST') { } // ── Write ──────────────────────────────────────────────────────────────────────────────────── +// ── Readiness: would a failover actually work right now ───────────────────────────────────── +// Every row is a deterministic check with a stated basis. The assistant on this page EXPLAINS +// these rows; it never produces them. A model must not be the thing that says failover is ready — +// that is precisely the class of answer this codebase keeps finding to be confidently wrong, and +// on 2026-08-23 the coverage card itself was the confidently wrong surface: 12 containers listed, +// none of them present on the partner. +// +// Rows are ordered by what breaks first, not by severity, so reading top to bottom follows the +// order a real outage would hit them. +if (($_POST['action'] ?? '') === 'readiness') { + $rows = []; + $add = function (string $id, string $label, string $verdict, string $detail, string $ask = '') + use (&$rows) { + // verdict: ok | warn | fail | unknown — unknown is never dressed up as ok + $rows[] = ['id' => $id, 'label' => $label, 'verdict' => $verdict, + 'detail' => $detail, 'ask' => $ask]; + }; + + $me = vv_detect_host(); + $ME = strtoupper($me); + $conf = vv_read_conf_raw('master.conf'); + $hc = vv_read_conf_raw($me . '.conf'); + + // 1. is fallback even armed + $fbEnabled = preg_match('/^\s*FALLBACK_ENABLED\s*=\s*"?(\w+)/m', $conf, $m) ? $m[1] : 'unset'; + $add('enabled', 'Fallback armed', + $fbEnabled === 'true' ? 'ok' : 'fail', + 'FALLBACK_ENABLED=' . $fbEnabled, + 'FALLBACK_ENABLED is ' . $fbEnabled . ' — what does that mean for a real outage?'); + + // 2. current state — anything but NORMAL means it is already doing something + $stateFile = STATE_DIR . '/fallback_state.db'; + $state = 'unknown'; + if (is_readable($stateFile) && preg_match('/^state=(\S+)/m', (string)@file_get_contents($stateFile), $m)) { + $state = $m[1]; + } + // Same rule the node cards use: no state file plus a live daemon means the node has simply + // never transitioned, which is health, not ignorance. Reading the file alone gives a healthy + // node the same verdict as one whose daemon is dead. + $daemon = function_exists('vv_fb_proc') ? (vv_fb_proc('fallback')['running'] ?? false) : false; + $inferred = false; + if ($state === 'unknown' && $daemon) { $state = 'NORMAL'; $inferred = true; } + + $add('state', 'State is NORMAL', + $state === 'NORMAL' ? 'ok' : ($state === 'unknown' ? 'unknown' : 'warn'), + 'state=' . $state . ($inferred ? ' (from the live daemon — never transitioned)' : ''), + 'Fallback state is ' . $state . '. What does that mean and what should I check?'); + + // 3. coverage configured at all + $covered = []; + for ($t = 1; $t <= 4; $t++) { + foreach (vv_parse_conf_list($hc, "FALLBACK_{$ME}_TIER{$t}") as $c) $covered[] = $c; + } + $add('coverage', 'Containers are covered', + $covered ? 'ok' : 'fail', + $covered ? count($covered) . ' container(s) across the tiers' : 'no containers in any tier', + $covered + ? 'Walk me through what happens if this host goes dark right now, tier by tier, with the delays.' + : 'Nothing is listed in my fallback tiers — what would happen if this host went dark?'); + + // 4. THE one that was silently false — does the partner actually hold them + $cache = (defined('VV_CACHE_ROOT') ? VV_CACHE_ROOT : '/tmp/varaverk') . '/api/fallback_presence.json'; + if (!is_readable($cache)) { + $add('present', 'Partner has the containers', 'unknown', + 'never checked — run the presence check', + 'How do I find out whether the partner actually has my covered containers?'); + } else { + $j = json_decode((string)@file_get_contents($cache), true); + $miss = (array)($j['missing'] ?? []); + $age = time() - (int)@filemtime($cache); + $when = $age < 3600 ? round($age / 60) . 'm ago' : round($age / 3600) . 'h ago'; + $add('present', 'Partner has the containers', + $miss ? 'fail' : 'ok', + $miss ? count($miss) . ' of ' . count($covered) . ' missing (' . $when . '): ' + . implode(', ', array_slice($miss, 0, 4)) . (count($miss) > 4 ? '…' : '') + : 'all ' . count($covered) . ' present (' . $when . ')', + $miss ? 'The partner is missing ' . implode(', ', array_slice($miss, 0, 6)) + . '. What happens during a failover, and how do I fix it?' : ''); + } + + // 5. host-specific networks that cannot be recreated on the partner + $wg = []; + foreach ($covered as $c) { + foreach (glob('/boot/config/plugins/dockerMan/templates-user/*.xml') as $x) { + $t = @file_get_contents($x); + if ($t === false || strpos($t, "$c") === false) continue; + if (preg_match('~(wg\d+)~', $t, $m)) $wg[] = "$c ({$m[1]})"; + break; + } + } + if ($wg) { + $add('wgnet', 'No tunnel-bound networks', 'warn', + implode(', ', $wg), + 'Some covered containers use a WireGuard-backed network. Why can that not move to the partner?'); + } + + // 6. handback writeback — invisible until the day it matters + $wb = preg_match('/^\s*FALLBACK_RSYNC_ENABLED\s*=\s*"?(\w+)/m', $conf, $m) ? $m[1] : 'unset'; + $add('writeback', 'Handback writeback', $wb === 'true' ? 'ok' : 'warn', + 'FALLBACK_RSYNC_ENABLED=' . $wb, + 'FALLBACK_RSYNC_ENABLED is ' . $wb . ' — what do I lose on handback?'); + + // Overall verdict is the worst row, never an average. One failed check is a failed failover. + $order = ['ok' => 0, 'warn' => 1, 'unknown' => 2, 'fail' => 3]; + $worst = 'ok'; + foreach ($rows as $r) if ($order[$r['verdict']] > $order[$worst]) $worst = $r['verdict']; + + echo json_encode(['ok' => true, 'verdict' => $worst, 'rows' => $rows, + 'summary' => $worst === 'ok' + ? 'Every check passed' + : ($worst === 'fail' ? 'A failover would NOT work as configured' + : 'Failover is configured but something needs a look')]); + exit; +} +// ── Push / remove / status: what the PARTNER actually holds ────────────────────────────────── +// Coverage names a container; fallback.sh starts it with `docker start`, which fails unless the +// partner already has it built. Measured 2026-08-23: 12 of 12 covered containers were absent from +// the partner, so every tier would have failed on the first real outage. These three actions are +// how the card closes and inspects that gap. +// +// Deliberately NOT folded into `cover`. Saving a tier list is a cheap, reversible config write; +// deploying a dozen containers onto another machine is neither, and a stray click should not be +// able to do it. +$_covAction = $_POST['action'] ?? ''; +if (in_array($_covAction, ['push', 'remove', 'deploy_status'], true)) { + $dir = rtrim(SCRIPTS_DIR, '/'); + $script = $dir . '/Fallback/coverage_deploy.sh'; + $runner = $dir . '/Plugin/unraid/run_job.sh'; + + if (!is_file($script)) { + echo json_encode(['ok' => false, 'error' => 'coverage_deploy.sh not found on this host']); + exit; + } + + // Status is read-only and fast enough to answer inline; the two that change the partner are + // dispatched to run_job.sh so they get a job record, a log, and a UI surface like every other + // long operation here. + if ($_covAction === 'deploy_status') { + $out = []; + exec('timeout 120 /bin/bash ' . escapeshellarg($script) . ' --status 2>&1', $out, $rc); + $present = []; $missing = []; + foreach ($out as $line) { + if (preg_match('/^\s{2}(\S+)\s+MISSING on/', $line, $m)) $missing[] = $m[1]; + elseif (preg_match('/^\s{2}(\S+)\s+on \S+ \((\w+)\)/', $line, $m)) $present[$m[1]] = $m[2]; + } + echo json_encode([ + 'ok' => true, + 'present' => $present, + 'missing' => $missing, + // rc 2 means "ran fine, some are missing" — not a failure of the check itself. + 'checked' => ($rc === 0 || $rc === 2), + ]); + exit; + } + + if (!is_file($runner)) { + echo json_encode(['ok' => false, 'error' => 'run_job.sh not found on this host']); + exit; + } + $flag = $_covAction === 'push' ? '--push' : '--remove'; + $stat = '/var/log/varaverk/Fallback/coverage_deploy.json'; + + shell_exec('setsid /bin/bash ' . escapeshellarg($runner) + . ' ' . escapeshellarg('Fallback/coverage_deploy.sh') + . ' ' . escapeshellarg($script) + . ' --manual ' . escapeshellarg($flag) + . ' >/dev/null 2>&1 true, 'status' => 'running', 'action' => $_covAction]); + exit; + } + } + usleep(250000); + } + echo json_encode(['ok' => false, + 'error' => 'Job did not report as running — check the Fallback log. It refuses to run unless fallback state is NORMAL.']); + exit; +} if (($_POST['action'] ?? '') !== 'cover') { echo json_encode(['ok' => false, 'error' => 'Unknown action']); exit; diff --git a/Plugin/unraid/include/ai.php b/Plugin/unraid/include/ai.php index a4449b2..256817a 100644 --- a/Plugin/unraid/include/ai.php +++ b/Plugin/unraid/include/ai.php @@ -1483,6 +1483,114 @@ function vv_ai_bug_report(array $b): string { // Strictly read-only, and there is no counterpart that changes any of it. Knowing a container is // down is what lets an explanation be about this machine instead of about Unraid in general; // restarting it is a decision that belongs to a person looking at the screen. +// ══════════════════════════════════════════════════════════════════════════════════════════════ +// Fallback readiness, for the assistant on the Fallback tab. +// +// Fallback differs from every other subsystem here in one way that shapes this whole function: it +// is DORMANT until it isn't. A watchdog leaves strikes and restarts to reason about; fallback +// leaves nothing at all until a real outage, so "looks fine" and "would work" are unrelated. On +// 2026-08-23 the coverage card showed 12 containers configured and every one of them was absent +// from the partner — a failover would have started nothing, and no surface said so. +// +// So this reports what would ACTUALLY happen, not what is configured to happen, and it is explicit +// about the difference between the two. +// +// Never blocks on the network. Partner presence costs an SSH round trip per container, which is far +// too slow for a question already waiting on a model, so it is read from the cache +// coverage_deploy.sh --status writes and reported WITH ITS AGE. A stale answer stated as stale is +// useful; a stale answer stated as current is the failure this whole feature exists to prevent. +// ══════════════════════════════════════════════════════════════════════════════════════════════ +function vv_ai_fallback_state(): string { + $me = vv_detect_host(); + if ($me === '') return ''; + // vv_detect_host() returns the LOWERCASE slug (host1); the conf keys are uppercase + // (FALLBACK_HOST1_TIER1). Building the key from the slug as-is silently matched nothing and + // reported "NOTHING is covered" on a host with twelve covered containers — a confidently + // wrong answer, which is the one outcome this block must never produce. + $ME = strtoupper($me); + + $conf = vv_read_conf_raw('master.conf'); + if ($conf === '') return ''; + + $s = "FALLBACK READINESS (read-only — you cannot change any of it, and you must never tell the " + . "operator a failover will work unless the evidence below says so)\n"; + + // ── current state ─────────────────────────────────────────────────────────────────────── + $stateFile = STATE_DIR . '/fallback_state.db'; + $state = 'unknown'; $since = ''; + if (is_readable($stateFile)) { + $raw = (string) @file_get_contents($stateFile); + if (preg_match('/^state=(\S+)/m', $raw, $m)) $state = $m[1]; + if (preg_match('/^fallback_start=(\d+)/m', $raw, $m) && (int)$m[1] > 0) { + $since = ' since ' . date('Y-m-d H:i', (int) $m[1]); + } + $age = time() - (int) @filemtime($stateFile); + // The steady NORMAL path writes nothing, so an old mtime is not staleness — it is quiet. + $s .= "- state: $state$since (state file last written " + . ($age < 3600 ? round($age / 60) . ' minutes' : round($age / 86400) . ' days') . " ago; " + . "the NORMAL path writes nothing, so an old file means nothing has changed)\n"; + } else { + // No file is not the same as not running: fallback.sh writes only on a transition. + $live = function_exists('vv_fb_proc') ? (vv_fb_proc('fallback')['running'] ?? false) : false; + $s .= $live + ? "- state: NORMAL (inferred — the daemon is running and has never recorded a transition, " + . "so it has written no state file; this is health, not ignorance)\n" + : "- state: no state file AND no running daemon — fallback is not operating on this host\n"; + } + + foreach (['FALLBACK_ENABLED', 'FALLBACK_RSYNC_ENABLED'] as $k) { + if (preg_match('/^\s*' . $k . '\s*=\s*"?(\w+)"?/m', $conf, $m)) { + $s .= "- $k: {$m[1]}" + . ($k === 'FALLBACK_RSYNC_ENABLED' && $m[1] !== 'true' + ? " <- handback writeback is OFF: anything the partner writes while covering " + . "for this host never comes home\n" : "\n"); + } + } + + // ── coverage, tier by tier, with the real delays ───────────────────────────────────────── + $hostConf = vv_read_conf_raw($me . '.conf'); + $covered = []; + for ($t = 1; $t <= 4; $t++) { + $names = vv_parse_conf_list($hostConf, "FALLBACK_{$ME}_TIER{$t}"); + if (!$names) continue; + $delay = ''; + if ($t > 1 && preg_match('/^\s*' . $ME . '_TIER' . $t . '_DELAY\s*=\s*"?(\d+)/m', $hostConf, $m)) { + $delay = " after {$m[1]} minutes"; + } + $s .= "- tier $t" . ($t === 1 ? ' (immediate)' : $delay) . ': ' . implode(', ', $names) . "\n"; + foreach ($names as $n) $covered[] = $n; + } + if (!$covered) { + $s .= "- coverage: NOTHING is covered — a failover would start no containers at all\n"; + return $s . "\n"; + } + + // ── does the partner actually have them ───────────────────────────────────────────────── + $cache = '/tmp/varaverk/api/fallback_presence.json'; + if (is_readable($cache)) { + $j = json_decode((string) @file_get_contents($cache), true); + $age = time() - (int) @filemtime($cache); + $miss = (array) ($j['missing'] ?? []); + $have = array_keys((array) ($j['present'] ?? [])); + $when = $age < 3600 ? round($age / 60) . ' minutes ago' : round($age / 3600) . ' hours ago'; + if ($miss) { + $s .= "- ON THE PARTNER (checked $when): " . count($miss) . ' of ' . count($covered) + . " covered container(s) DO NOT EXIST there: " . implode(', ', $miss) . "\n" + . " fallback.sh starts a covered container with `docker start`; it never creates one, " + . "so each of those would fail during a real outage. Push them from the Fallback " + . "coverage card.\n"; + } else { + $s .= "- ON THE PARTNER (checked $when): all " . count($have) + . " covered container(s) exist there\n"; + } + } else { + $s .= "- ON THE PARTNER: not checked. Say so plainly — whether a failover would actually " + . "start anything is UNKNOWN until the coverage card's presence check runs.\n"; + } + + return $s . "\n"; +} + function vv_ai_system_state(): string { $p = '/tmp/varaverk/api/monitor.json'; if (!is_readable($p)) return ''; diff --git a/Plugin/unraid/include/ai_profiles.php b/Plugin/unraid/include/ai_profiles.php index 155107e..00ec06c 100644 --- a/Plugin/unraid/include/ai_profiles.php +++ b/Plugin/unraid/include/ai_profiles.php @@ -116,7 +116,7 @@ const VV_AI_PROFILES_DEF = [ // system_state is read-only and shared with repair. Both need to know a container is down // or a pool is full to explain anything about this machine rather than about Unraid in // general; neither gets a way to act on it, and only repair can change a setting. - 'caps' => ['retrieve', 'health', 'system_state', 'run_evidence', 'scoped_log', + 'caps' => ['retrieve', 'health', 'system_state', 'fallback_state', 'run_evidence', 'scoped_log', 'incidents', 'conf_lookup', 'file_bugs'], ], // The only profile that may change a setting, and the only one not offered as a button. @@ -136,7 +136,7 @@ const VV_AI_PROFILES_DEF = [ 'hint' => 'Works through a finding with you, and can apply a fix you approve.', 'turns' => 3, 'ui' => false, - 'caps' => ['retrieve', 'health', 'system_state', 'run_evidence', 'scoped_log', 'incidents', + 'caps' => ['retrieve', 'health', 'system_state', 'fallback_state', 'run_evidence', 'scoped_log', 'incidents', 'conf_lookup', 'conf_write', 'probe', 'file_findings', 'phrasebook', 'past_fixes'], ], ]; @@ -148,6 +148,7 @@ const VV_AI_CAP_MEANING = [ 'kind_filter' => 'the retrieval kind filter the page exposes', 'health' => 'live health sweep measured at question time — the AI subsystem only', 'system_state' => 'read-only view of the machine: hardware, containers, pools, array, UPS', + 'fallback_state' => 'whether a failover would actually work: state, tiers, and whether the partner really has the covered containers', 'run_evidence' => 'run record and log tail for a script named in the question', 'scoped_log' => 'log tail for whatever the operator currently has open', 'incidents' => 'operator-written history of what previously went wrong with this thing', diff --git a/Plugin/unraid/include/fallback.php b/Plugin/unraid/include/fallback.php index 712e006..ad9312a 100644 --- a/Plugin/unraid/include/fallback.php +++ b/Plugin/unraid/include/fallback.php @@ -73,6 +73,8 @@ function vv_fb_parse_state(string $text): array { 'tier4_started' => false, 'partnership_suspended' => false, 'partner_lost_at' => 0, + // Set only when the verdict came from the daemon rather than the file — see vv_fb_all(). + 'inferred' => false, ]; foreach (explode("\n", $text) as $line) { $line = trim($line); @@ -313,6 +315,29 @@ function vv_fb_all(): array { : vv_fb_remote_dryrun_state($ip, $mySshKey, (int)$proc['pid']); } + // ── A live daemon that has simply never transitioned ──────────────────────────────── + // fallback.sh writes its state file ONLY on a transition; the steady NORMAL path writes + // nothing at all. So a node that has run cleanly since it was built has no file, and + // reading the file alone reports it as UNKNOWN — the same verdict given to a node whose + // daemon is dead. Those are opposite conditions and they were rendered identically. + // + // Observed on HOST2 2026-08-23: daemon live (pid 2208657, valid lock), tailscale and ssh + // both fine, no state file, card said UNKNOWN. + // + // The rule this page is built on — never claim healthy for a host you cannot verify — is + // kept: this host CAN be verified, just not from the file that was being consulted. The + // daemon holding a live lock is the evidence. reach['state_file'] still reports false, + // because there genuinely is no file; 'inferred' says where the verdict came from instead. + // Captured before the inference below rewrites it: reach[state_file] must keep answering + // "was there a file", not "do we have a verdict". Deriving it after inference made the + // card claim a state file existed on a host that has none. + $hadStateFile = ($state['state'] ?? 'UNKNOWN') !== 'UNKNOWN'; + + if (($state['state'] ?? 'UNKNOWN') === 'UNKNOWN' && ($proc['running'] ?? null) === true) { + $state['state'] = 'NORMAL'; + $state['inferred'] = true; + } + // How fresh the state actually is. The daemon rewrites its file every check interval, so // an age far past that interval means it is wedged even while the process still exists. $stateAge = null; @@ -326,7 +351,7 @@ function vv_fb_all(): array { 'tailscale' => $isMe ? true : ($ts['online'] === true), 'ip' => $isMe ? null : $ip, 'ssh' => $isMe ? true : ($running !== [] || ($proc['running'] !== null)), - 'state_file' => ($state['state'] ?? 'UNKNOWN') !== 'UNKNOWN', + 'state_file' => $hadStateFile, ]; $nodes[] = [ diff --git a/Plugin/unraid/pages/fallback.php b/Plugin/unraid/pages/fallback.php index a6ea79b..8112d1c 100644 --- a/Plugin/unraid/pages/fallback.php +++ b/Plugin/unraid/pages/fallback.php @@ -64,6 +64,28 @@ if (vv_ai_ui_on()) vv_ai_chat_assets(); border-radius:4px;padding:6px 9px;margin-bottom:8px; } .vv-fb-tierhdr-t { font-size:12px;font-weight:700;color:#ffb74d;letter-spacing:.01em; } .vv-fb-tierhdr-s { font-size:10px;color:#7a6038; } +/* ── Failover readiness ────────────────────────────────────────────────────────────────────── + Every other card on this page shows what is CONFIGURED. This one shows what would actually + happen, which on 2026-08-23 turned out to be a different thing entirely — twelve containers + configured, none of them present on the partner, and no surface said so. + Verdict colour always ships beside a word, never alone. */ +.vv-fb-rd { display:flex;flex-direction:column;gap:4px; } +.vv-fb-rdrow { display:flex;align-items:center;gap:9px;padding:5px 8px;border-radius:3px; + background:#0d0d0d;border:1px solid #161616;border-left:3px solid var(--rv,#333); } +.vv-fb-rdrow.ok { --rv:#4caf50; } +.vv-fb-rdrow.warn { --rv:#ffb74d; } +.vv-fb-rdrow.fail { --rv:#ef5350; } +.vv-fb-rdrow.unknown { --rv:#5a7a8a; } +.vv-fb-rdv { font-size:9px;font-weight:700;letter-spacing:.05em;text-transform:uppercase; + min-width:56px;color:var(--rv,#666); } +.vv-fb-rdl { font-size:12px;color:#b8b8b8;min-width:190px; } +.vv-fb-rdd { font-size:11px;color:#5a5a5a;flex:1;min-width:0;overflow:hidden; + text-overflow:ellipsis;white-space:nowrap; } +.vv-fb-rdwhy { font-size:10px;padding:2px 8px;border-radius:3px;cursor:pointer; + background:#0e1a2a;color:#7ab;border:1px solid #1e3a5a;white-space:nowrap; } +.vv-fb-rdwhy:hover { background:#12233a; } +.vv-fb-rdsum { font-size:11px;font-weight:600;margin-bottom:7px; } + /* ── Fallback coverage ── */ /* One continuum, worst outcome to best: never comes back → 24h → 12h → 4h → immediate → never goes down at all. The colour answers "how long am I without this if the partner takes over", @@ -267,6 +289,18 @@ if (vv_ai_ui_on()) vv_ai_chat_assets();
Loading…
+ +
+
+

Failover readiness

+ would a failover actually work right now + + +
+
checking…
+
+
+
+ On partner + checking… + + + + +
+
@@ -654,13 +704,14 @@ function _nodeCard(node, data) { ${node.is_me ? 'US' : ''} ${(node.is_me && node.proc && node.proc.running && node.proc.mode === 'live') ? _ptStatus(st) : ''} - ${_stateBadge(shown)}${dryRun ? 'preview' : ''} + ${_stateBadge(shown)}${dryRun ? 'preview' : ''}${(st.inferred && !dryRun) ? 'inferred' : ''}
${_leg(reach.tailscale, 'tailscale')} ${_leg(node.is_me ? true : reach.ssh, 'ssh')} ${_leg(reach.state_file, 'state file')} + ${st.inferred ? '· never transitioned' : ''} ${node.ts_ip ? `${vvEscHtml(node.ts_ip)}` : ''}
@@ -1081,9 +1132,174 @@ window.vvFbCovSave = async function () { fbEl.style.color = '#ef5350'; fbEl.textContent = 'Request failed: ' + e; }); }; + +function _vvFbPartnerName() { + const el = document.getElementById('vv-fb-cov-partner'); + const t = el ? el.textContent.trim() : ''; + return (t && t !== 'the partner') ? t : 'the partner'; +} + +// ── Deploy: what the partner actually holds ─────────────────────────────────────────────────── +// Presence is read from the partner, never inferred from the tier list. The whole point of this +// row is that the two disagree — coverage said 12 containers, the partner had none of them. +window.vvFbCovPresence = async function () { + const el = document.getElementById('vv-fb-cov-presence'); + if (!el) return; + try { + const fd = new URLSearchParams({ action: 'deploy_status' }); + const d = await (await fetch('/plugins/varaverk/api/fallback_coverage.php', + { method: 'POST', body: fd })).json(); + if (!d.ok || !d.checked) { el.style.color = '#a05a2c'; el.textContent = d.error || 'could not check'; return; } + const have = Object.keys(d.present || {}).length, miss = (d.missing || []).length; + if (miss === 0 && have === 0) { el.style.color = '#444'; el.textContent = 'nothing covered'; } + else if (miss === 0) { el.style.color = '#4caf50'; el.textContent = `all ${have} present`; } + else { + el.style.color = '#ef5350'; + // Named, not just counted: "3 missing" is a number, the names are what you act on. + el.textContent = `${miss} missing — ${(d.missing||[]).slice(0,3).join(', ')}${miss>3?` +${miss-3}`:''}`; + } + } catch (e) { el.style.color = '#a05a2c'; el.textContent = 'check failed'; } +}; + +window.vvFbCovDeploy = async function (which) { + const push = which === 'push'; + const fbEl = document.getElementById('vv-fb-cov-dfb'); + const btn = document.getElementById(push ? 'vv-fb-cov-push' : 'vv-fb-cov-rm'); + const n = _vvFbCov ? Object.keys(_vvFbCov.cover).length : 0; + + const msg = push + ? `Build ${n} container${n!==1?'s':''} on ${_vvFbPartnerName()}?\n\n` + + 'Each is created and left STOPPED so a failover can start it. Nothing starts running now.\n\n' + + 'Save first if you have unsaved changes — this pushes what is in the conf, not what is on screen.' + : `Stop and remove ${n} container${n!==1?'s':''} on ${_vvFbPartnerName()} AND DELETE THEIR APPDATA?\n\n` + + 'Not reversible. Only paths under /mnt/*/appdata* are touched; a bind of the appdata root is refused.\n\n' + + 'If the partner has ever covered for this host, what it holds may be the NEWER copy — the one a ' + + 'handback rsyncs home. Fallback state is NORMAL, so nothing is failing over right now, but a ' + + 'handback that partly failed would not show up here.\n\n' + + 'Coverage stays as configured, so a later Push rebuilds the containers from scratch.'; + if (!await vvConfirm(msg, { title: push ? 'Push to partner' : 'Remove from partner', + confirmText: push ? 'Push' : 'Remove' })) return; + + btn.disabled = true; const label = btn.textContent; btn.textContent = push ? 'Pushing…' : 'Removing…'; + fbEl.style.color = '#7ab'; fbEl.textContent = 'job started…'; + try { + const fd = new URLSearchParams({ action: which }); + const d = await (await fetch('/plugins/varaverk/api/fallback_coverage.php', + { method: 'POST', body: fd })).json(); + if (!d.ok) { fbEl.style.color = '#ef5350'; fbEl.textContent = d.error || 'Failed'; } + else { + // The job runs past this response. Re-checking presence is the only honest completion + // signal available here, so poll it rather than claiming success on dispatch. + fbEl.style.color = '#7ab'; fbEl.textContent = 'running — see the Fallback log'; + let ticks = 0; + const t = setInterval(async () => { + await vvFbCovPresence(); + // Presence just changed, so the readiness verdict that depends on it is stale. + vvFbReadiness(false); + if (++ticks >= 20) { clearInterval(t); fbEl.textContent = ''; } + }, 6000); + } + } catch (e) { + fbEl.style.color = '#ef5350'; fbEl.textContent = 'Request failed: ' + e; + } + btn.disabled = false; btn.textContent = label; +}; +// ── Assistant ──────────────────────────────────────────────────────────────────────────────── +// vv_ai_chat_markup() above emits the boxes and nothing else — no