_TIER1-4 in this host's own conf — which of THIS host's // containers the partner starts when this host goes dark, and in which delay tier. // // OPERATIONAL MODEL // GET this host's containers plus current tier membership. // POST action=cover tiers= rewrites all four tier arrays. // // Originally the Coverage picker on the Partnership page (e8ee5b0), removed the same day in // 1a836da — "they describe what the partner runs during an outage, which is the Fallback tab's // subject" — and never rehomed, because that tab had nothing to host it. This is that card, // rebuilt where it belongs, with the services half left behind: pushing XML templates to a // mirror is an onboard concern, not a failover one. // // DESIGN PRINCIPLES // Edit the array fallback.sh actually reads, not a parallel one. // The coverage tiers already exist and already carry the timing. A second "what to fail // over" list would be a second answer to the same question, and the two would drift. // // A host edits only its OWN tiers, and the page says so. // FALLBACK__TIER* lives in that host's conf and describes what someone else runs for // it. Sparse checkout means this host does not have the partner's host*.conf at all — only // the read-only RAM cache conf_sync fills — so an editor for the partner's coverage would // be writing to a cache that the next sync overwrites. Configure HOST2's coverage from // HOST2. This is the same trap that left the Watchdog card reporting a partner's lists as // empty when they were merely somewhere else. // // OPERATIONAL SAFEGUARDS // POST only for writes, so Unraid's CSRF guard applies. // // Names are validated against containers this host runs, PLUS whatever the tiers already name. // The conf legitimately holds entries for containers not present right now — removed, // stopped, or renamed. Validating only against the running set would refuse to save a list // the operator never touched. New names still have to be real; the guard is against // inventing containers, not against keeping ones already recorded. // // A tier outside 1-4 is rejected, never clamped. Silently moving a container from tier 9 to // tier 4 would give it a 24-hour delay nobody asked for. // // An absent array is refused, not appended. Writing a new block into an unknown position in a // conf is how a setting ends up in the wrong section and stops being read. // // Writing an empty list is allowed — "cover nothing" is a legitimate choice and the only way // to express it. // // REQUEST // GET → current lists // POST action=cover tiers={"Emby":1,...} → rewrite tiers 1-4 // // RESPONSE // {"ok":true,...} read payload, or {"ok":true,"counts":{...}} after a write // {"ok":false,"error":string} validation or write failure, stated // // DEPENDS ON // include/confform.php vv_conf_edit(), vv_conf_last_error(), vv_parse_conf_list() // include/common.php vv_docker_containers() // include/config.php vv_detect_host(), vv_read_conf_raw(), vv_push_master_conf() // ═══════════════════════════════════════════════════════════════════════════════════════════════ header('Content-Type: application/json'); require_once dirname(__DIR__) . '/include/fallback.php'; // vv_fb_proc() require_once dirname(__DIR__) . '/include/confform.php'; require_once dirname(__DIR__) . '/include/common.php'; $hostId = vv_detect_host(); $hostUp = strtoupper($hostId); $myConf = $hostId . '.conf'; $TIERS = [1, 2, 3, 4]; $tierVar = fn(int $t) => "FALLBACK_{$hostUp}_TIER{$t}"; // ── Read ───────────────────────────────────────────────────────────────────────────────────── if ($_SERVER['REQUEST_METHOD'] !== 'POST') { $raw = vv_read_conf_raw($myConf); $mRaw = vv_read_conf_raw('master.conf'); $cover = []; foreach ($TIERS as $t) { // host*.conf first, master.conf second — installs that kept the tiers there still read. $vals = vv_parse_conf_list($raw, $tierVar($t)) ?: vv_parse_conf_list($mRaw, $tierVar($t)); foreach ($vals as $c) { $c = trim($c); if ($c !== '') $cover[$c] = $t; } } $containers = []; foreach (vv_docker_containers() as $c) { $n = is_array($c) ? ($c['name'] ?? '') : (string)$c; if ($n !== '') $containers[] = $n; } sort($containers, SORT_NATURAL | SORT_FLAG_CASE); // Stacks are declared by the PARTNERSHIP OWNER and deployed to everyone, so on a mirror they // are not in this host's conf at all — HOST2_PARTNERSHIP_AUTH_STACK is the shipped template, // still commented out, while the eight auth containers it describes run there permanently // because the owner put them there. Reading "this host's" stacks left the mirror's card // showing every one of them as ordinary, selectable, uncovered. // // Owner's conf first, then this host's, unioned: on the owner the two are the same file, and // a host that declares extras of its own still has them honoured. The owner's copy reaches a // mirror through the conf_sync RAM cache, which vv_read_host_conf_raw() knows how to find. $ownerSlot = strtolower(vv_parse_conf_scalar(vv_read_conf_raw('master.conf'), 'PARTNERSHIP_OWNER_HOST')); $stackSrc = []; foreach (array_unique(array_filter([$ownerSlot, $hostId])) as $slot) { $stackSrc[strtoupper($slot)] = vv_read_host_conf_raw($slot); } $stackOf = []; $byLower = []; foreach ($containers as $n) $byLower[strtolower($n)] = $n; foreach ($stackSrc as $id => $srcRaw) { foreach ([ 'auth' => "{$id}_PARTNERSHIP_AUTH_STACK", 'arrs' => "{$id}_PARTNERSHIP_ARR_STACK", 'services' => "{$id}_PARTNERSHIP_SERVICES_STACK", ] as $label => $var) { foreach (vv_parse_conf_list($srcRaw, $var) as $xml) { $n = preg_replace('/^my-|\.xml$/', '', trim($xml)); if ($n === '') continue; // Only what this host actually runs. The owner's stack lists everything it // deploys mesh-wide; a name with no container here is not "always up" here. if (!isset($byLower[strtolower($n)])) continue; $stackOf[$byLower[strtolower($n)]] = $label; } } } // Named in a tier but not installed here. Reported rather than filtered: a tier entry for a // container that does not exist is a line fallback.sh fails on during an outage, which is // the worst possible moment to find a typo. $missing = []; $have = array_map('strtolower', $containers); foreach (array_keys($cover) as $n) if (!in_array(strtolower($n), $have, true)) $missing[] = $n; echo json_encode([ 'ok' => true, 'host' => $hostUp, 'containers' => $containers, 'cover' => (object)$cover, 'stacks' => (object)$stackOf, 'missing' => $missing, 'tier_vars' => array_map($tierVar, $TIERS), ]); exit; } // ── Write ──────────────────────────────────────────────────────────────────────────────────── // ── Readiness: would a failover actually work right now ───────────────────────────────────── // Every row is a deterministic check with a stated basis. The assistant on this page EXPLAINS // these rows; it never produces them. A model must not be the thing that says failover is ready — // that is precisely the class of answer this codebase keeps finding to be confidently wrong, and // on 2026-08-23 the coverage card itself was the confidently wrong surface: 12 containers listed, // none of them present on the partner. // // Rows are ordered by what breaks first, not by severity, so reading top to bottom follows the // order a real outage would hit them. if (($_POST['action'] ?? '') === 'readiness') { $rows = []; $add = function (string $id, string $label, string $verdict, string $detail, string $ask = '') use (&$rows) { // verdict: ok | warn | fail | unknown — unknown is never dressed up as ok $rows[] = ['id' => $id, 'label' => $label, 'verdict' => $verdict, 'detail' => $detail, 'ask' => $ask]; }; $me = vv_detect_host(); $ME = strtoupper($me); $conf = vv_read_conf_raw('master.conf'); $hc = vv_read_conf_raw($me . '.conf'); // 1. is fallback even armed $fbEnabled = preg_match('/^\s*FALLBACK_ENABLED\s*=\s*"?(\w+)/m', $conf, $m) ? $m[1] : 'unset'; $add('enabled', 'Fallback armed', $fbEnabled === 'true' ? 'ok' : 'fail', 'FALLBACK_ENABLED=' . $fbEnabled, 'FALLBACK_ENABLED is ' . $fbEnabled . ' — what does that mean for a real outage?'); // 2. current state — anything but NORMAL means it is already doing something $stateFile = STATE_DIR . '/fallback_state.db'; $state = 'unknown'; if (is_readable($stateFile) && preg_match('/^state=(\S+)/m', (string)@file_get_contents($stateFile), $m)) { $state = $m[1]; } // Same rule the node cards use: no state file plus a live daemon means the node has simply // never transitioned, which is health, not ignorance. Reading the file alone gives a healthy // node the same verdict as one whose daemon is dead. $daemon = function_exists('vv_fb_proc') ? (vv_fb_proc('fallback')['running'] ?? false) : false; $inferred = false; if ($state === 'unknown' && $daemon) { $state = 'NORMAL'; $inferred = true; } $add('state', 'State is NORMAL', $state === 'NORMAL' ? 'ok' : ($state === 'unknown' ? 'unknown' : 'warn'), 'state=' . $state . ($inferred ? ' (from the live daemon — never transitioned)' : ''), 'Fallback state is ' . $state . '. What does that mean and what should I check?'); // 3. coverage configured at all $covered = []; for ($t = 1; $t <= 4; $t++) { foreach (vv_parse_conf_list($hc, "FALLBACK_{$ME}_TIER{$t}") as $c) $covered[] = $c; } $add('coverage', 'Containers are covered', $covered ? 'ok' : 'fail', $covered ? count($covered) . ' container(s) across the tiers' : 'no containers in any tier', $covered ? 'Walk me through what happens if this host goes dark right now, tier by tier, with the delays.' : 'Nothing is listed in my fallback tiers — what would happen if this host went dark?'); // 4. THE one that was silently false — does the partner actually hold them $cache = (defined('VV_CACHE_ROOT') ? VV_CACHE_ROOT : '/tmp/varaverk') . '/api/fallback_presence.json'; if (!is_readable($cache)) { $add('present', 'Partner has the containers', 'unknown', 'never checked — run the presence check', 'How do I find out whether the partner actually has my covered containers?'); } else { $j = json_decode((string)@file_get_contents($cache), true); $miss = (array)($j['missing'] ?? []); $age = time() - (int)@filemtime($cache); $when = $age < 3600 ? round($age / 60) . 'm ago' : round($age / 3600) . 'h ago'; $add('present', 'Partner has the containers', $miss ? 'fail' : 'ok', $miss ? count($miss) . ' of ' . count($covered) . ' missing (' . $when . '): ' . implode(', ', array_slice($miss, 0, 4)) . (count($miss) > 4 ? '…' : '') : 'all ' . count($covered) . ' present (' . $when . ')', $miss ? 'The partner is missing ' . implode(', ', array_slice($miss, 0, 6)) . '. What happens during a failover, and how do I fix it?' : ''); } // 5. host-specific networks that cannot be recreated on the partner $wg = []; foreach ($covered as $c) { foreach (glob('/boot/config/plugins/dockerMan/templates-user/*.xml') as $x) { $t = @file_get_contents($x); if ($t === false || strpos($t, "$c") === false) continue; if (preg_match('~(wg\d+)~', $t, $m)) $wg[] = "$c ({$m[1]})"; break; } } if ($wg) { $add('wgnet', 'No tunnel-bound networks', 'warn', implode(', ', $wg), 'Some covered containers use a WireGuard-backed network. Why can that not move to the partner?'); } // 6. handback writeback — invisible until the day it matters $wb = preg_match('/^\s*FALLBACK_RSYNC_ENABLED\s*=\s*"?(\w+)/m', $conf, $m) ? $m[1] : 'unset'; $add('writeback', 'Handback writeback', $wb === 'true' ? 'ok' : 'warn', 'FALLBACK_RSYNC_ENABLED=' . $wb, 'FALLBACK_RSYNC_ENABLED is ' . $wb . ' — what do I lose on handback?'); // Overall verdict is the worst row, never an average. One failed check is a failed failover. $order = ['ok' => 0, 'warn' => 1, 'unknown' => 2, 'fail' => 3]; $worst = 'ok'; foreach ($rows as $r) if ($order[$r['verdict']] > $order[$worst]) $worst = $r['verdict']; echo json_encode(['ok' => true, 'verdict' => $worst, 'rows' => $rows, 'summary' => $worst === 'ok' ? 'Every check passed' : ($worst === 'fail' ? 'A failover would NOT work as configured' : 'Failover is configured but something needs a look')]); exit; } // ── Push / remove / status: what the PARTNER actually holds ────────────────────────────────── // Coverage names a container; fallback.sh starts it with `docker start`, which fails unless the // partner already has it built. Measured 2026-08-23: 12 of 12 covered containers were absent from // the partner, so every tier would have failed on the first real outage. These three actions are // how the card closes and inspects that gap. // // Deliberately NOT folded into `cover`. Saving a tier list is a cheap, reversible config write; // deploying a dozen containers onto another machine is neither, and a stray click should not be // able to do it. $_covAction = $_POST['action'] ?? ''; if (in_array($_covAction, ['push', 'remove', 'deploy_status'], true)) { $dir = rtrim(SCRIPTS_DIR, '/'); $script = $dir . '/Fallback/coverage_deploy.sh'; $runner = $dir . '/Plugin/unraid/run_job.sh'; if (!is_file($script)) { echo json_encode(['ok' => false, 'error' => 'coverage_deploy.sh not found on this host']); exit; } // Status is read-only and fast enough to answer inline; the two that change the partner are // dispatched to run_job.sh so they get a job record, a log, and a UI surface like every other // long operation here. if ($_covAction === 'deploy_status') { $out = []; exec('timeout 120 /bin/bash ' . escapeshellarg($script) . ' --status 2>&1', $out, $rc); $present = []; $missing = []; foreach ($out as $line) { if (preg_match('/^\s{2}(\S+)\s+MISSING on/', $line, $m)) $missing[] = $m[1]; elseif (preg_match('/^\s{2}(\S+)\s+on \S+ \((\w+)\)/', $line, $m)) $present[$m[1]] = $m[2]; } echo json_encode([ 'ok' => true, 'present' => $present, 'missing' => $missing, // rc 2 means "ran fine, some are missing" — not a failure of the check itself. 'checked' => ($rc === 0 || $rc === 2), ]); exit; } if (!is_file($runner)) { echo json_encode(['ok' => false, 'error' => 'run_job.sh not found on this host']); exit; } $flag = $_covAction === 'push' ? '--push' : '--remove'; $stat = '/var/log/varaverk/Fallback/coverage_deploy.json'; shell_exec('setsid /bin/bash ' . escapeshellarg($runner) . ' ' . escapeshellarg('Fallback/coverage_deploy.sh') . ' ' . escapeshellarg($script) . ' --manual ' . escapeshellarg($flag) . ' >/dev/null 2>&1 true, 'status' => 'running', 'action' => $_covAction]); exit; } } usleep(250000); } echo json_encode(['ok' => false, 'error' => 'Job did not report as running — check the Fallback log. It refuses to run unless fallback state is NORMAL.']); exit; } if (($_POST['action'] ?? '') !== 'cover') { echo json_encode(['ok' => false, 'error' => 'Unknown action']); exit; } $known = []; foreach (vv_docker_containers() as $c) { $n = is_array($c) ? ($c['name'] ?? '') : (string)$c; if ($n !== '') $known[strtolower($n)] = $n; } $existingRaw = vv_read_conf_raw($myConf); foreach ($TIERS as $t) { foreach (vv_parse_conf_list($existingRaw, $tierVar($t)) as $n) { $n = trim($n); if ($n !== '' && !isset($known[strtolower($n)])) $known[strtolower($n)] = $n; } } // Stack containers refused here, not only greyed out in the picker. A disabled select is a // courtesy to the operator, not a constraint on the endpoint. // // Sourced from the OWNER's conf as well as this host's, for the same reason the read path is: on // a mirror the stack it runs is the owner's declaration, and checking only the local conf would // have let a mirror assign a fallback tier to a container that never stops. $ownerSlotW = strtolower(vv_parse_conf_scalar(vv_read_conf_raw('master.conf'), 'PARTNERSHIP_OWNER_HOST')); $stackNames = []; foreach (array_unique(array_filter([$ownerSlotW, $hostId])) as $slot) { $srcRaw = vv_read_host_conf_raw($slot); $id = strtoupper($slot); foreach ([ "{$id}_PARTNERSHIP_AUTH_STACK", "{$id}_PARTNERSHIP_ARR_STACK", "{$id}_PARTNERSHIP_SERVICES_STACK", ] as $var) { foreach (vv_parse_conf_list($srcRaw, $var) as $xml) { $n = preg_replace('/^my-|\.xml$/', '', trim($xml)); if ($n !== '') $stackNames[strtolower($n)] = true; } } } $map = json_decode((string)($_POST['tiers'] ?? ''), true); if (!is_array($map)) { echo json_encode(['ok' => false, 'error' => 'tiers must be an object']); exit; } $byTier = array_fill_keys($TIERS, []); foreach ($map as $name => $tier) { $t = (int)$tier; if (!in_array($t, $TIERS, true)) { echo json_encode(['ok' => false, 'error' => "Tier $tier is not 1-4 (for $name)"]); exit; } if (!isset($known[strtolower((string)$name)])) { echo json_encode(['ok' => false, 'error' => "No container named $name on this host"]); exit; } if (isset($stackNames[strtolower((string)$name)])) { // Direction-neutral wording: on the owner this container is deployed TO the partner, on a // mirror it was deployed HERE by the owner. Both mean the same thing for coverage — it // runs on both nodes continuously, so there is nothing for a tier to start. echo json_encode(['ok' => false, 'error' => "$name belongs to a partnership stack — it runs on both nodes continuously, so it cannot be given a fallback tier"]); exit; } $byTier[$t][] = $known[strtolower((string)$name)]; } // Rewrites one `NAME=(` … `)` block in place, preserving the conf's leading indent. $rewrite = function (string $cur, string $var, array $items): ?string { $body = ''; foreach ($items as $i) $body .= " \"" . $i . "\"\n"; $pattern = '/^([ \t]*)' . preg_quote($var, '/') . '=\((?:[^)]*)\)/m'; if (!preg_match($pattern, $cur)) return null; // absent: refuse rather than append blind return preg_replace_callback($pattern, fn($m) => $m[1] . $var . "=(\n" . $body . $m[1] . ")", $cur, 1); }; $ok = vv_conf_edit($myConf, function (string $cur) use ($byTier, $TIERS, $tierVar, $rewrite): ?string { foreach ($TIERS as $t) { $next = $rewrite($cur, $tierVar($t), $byTier[$t]); if ($next === null) return null; $cur = $next; } return $cur; }, [], array_map($tierVar, $TIERS)); if (!$ok) { echo json_encode(['ok' => false, 'error' => vv_conf_last_error() ?: 'Write failed']); exit; } // A partner holding the old list is a partner that will act on the old list. vv_push_master_conf(); echo json_encode(['ok' => true, 'counts' => array_map('count', $byTier)]);