diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index dd81d6b..53a3356 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -122,6 +122,44 @@ STATE_DIR="/boot/config/plugins/varaverk/State_Files" PERSISTENT_CONF_CACHE="/boot/config/plugins/varaverk/.cache/vv/d" +# ── Cache Roots ── +# Everything Varaverk keeps in RAM, under one root, defined once. +# +# These used to be four literals in load_config.sh and three more in the PHP layer, spread over +# /tmp/vv_cache, /tmp/arr_cache, /tmp/.cache/vv/d, /tmp/.cache/vv/ai and /tmp/varaverk_ai_jobs — +# five naming schemes, and no single place that could tell you what Varaverk had in tmpfs. PHP +# could not read load_config.sh, so it restated the paths it needed and the two layers were kept +# in agreement by hand. They live here because master.conf is the one file both layers actually +# read: bash sources it, and the PHP conf parser resolves ${VAR} against it the same way. +# +# ONE ROOT, NOT ONE DIRECTORY. The subdirectories are deliberately separate and must stay that +# way — they have genuinely different rules: +# +# conf/ holds partner credentials, is chmod 700, and IS snapshotted to PERSISTENT_CONF_CACHE +# so it survives a reboot. +# ai/ holds non-secret token counters that are worthless when stale and must NOT be +# preserved across a reboot — losing them means "not collected here" until the next +# sync, which is the honest answer. +# jobs/ holds in-flight work handed off to a detached worker — an AI answer being generated, +# a container action being applied. chmod 700: each is readable by anyone who can guess +# its token, which is why the tokens are random_bytes and not sequential. +# arr/ is the large one (100s of MB) and is restored from a DATA_DIR backup on demand. +# api/ is the WebGUI payload cache, and is the only one safe to delete at any moment. +# +# Collapsing those into a single directory would give the credentials the token cache's +# persistence rules, or the reverse. The win here is one definition, not one folder. +# +# Point VV_CACHE_ROOT somewhere else and everything follows. It must be on a filesystem that is +# cleared or safe to clear on boot — every consumer treats a missing cache as a cold start, and +# nothing here is a source of truth for anything. + VV_CACHE_ROOT="/tmp/varaverk" + VV_CACHE_DIR="${VV_CACHE_ROOT}/api" # WebGUI payload cache — monitor, arrs, ai + CONF_RAM_CACHE_DIR="${VV_CACHE_ROOT}/conf" # partner host*.conf — chmod 700, snapshotted + ARR_CACHE_DIR="${VV_CACHE_ROOT}/arr" # arr payloads — restored from DATA_DIR on demand + AI_TOKEN_CACHE_DIR="${VV_CACHE_ROOT}/ai" # partner token ledgers — never preserved + AI_JOB_DIR="${VV_CACHE_ROOT}/jobs/ai" # in-flight AI answers — chmod 700 + DOCKER_JOB_DIR="${VV_CACHE_ROOT}/jobs/docker" # in-flight container actions — chmod 700 + # ── Version Parity ── # Controls behaviour when local and remote unRAID versions differ. # Major version mismatch always aborts regardless of this setting. diff --git a/Plugin/unraid/api/docker_action.php b/Plugin/unraid/api/docker_action.php index 34da5eb..7b4b408 100644 --- a/Plugin/unraid/api/docker_action.php +++ b/Plugin/unraid/api/docker_action.php @@ -95,7 +95,9 @@ header('Content-Type: application/json'); // take, because the operator's next move is to try again on a machine that already did it. require_once dirname(__DIR__) . '/include/config.php'; -define('VV_JOB_DIR', '/tmp/varaverk_dk_jobs'); +// VV_JOB_DIR is defined in config.php with every other tmpfs path, read from DOCKER_JOB_DIR in +// master.conf. It was declared here, inside an endpoint, which is why a survey of Varaverk's +// cache and job directories missed it entirely. $action = trim($_POST['action'] ?? ''); $name = trim($_POST['name'] ?? ''); diff --git a/Plugin/unraid/include/ai.php b/Plugin/unraid/include/ai.php index ee1952a..6a42688 100644 --- a/Plugin/unraid/include/ai.php +++ b/Plugin/unraid/include/ai.php @@ -79,7 +79,8 @@ require_once __DIR__ . '/config.php'; -define('VV_AI_JOB_DIR', '/tmp/varaverk_ai_jobs'); +// VV_AI_JOB_DIR and VV_AI_TOKEN_CACHE_DIR are defined in config.php with every other cache path, +// read from master.conf so this layer and load_config.sh cannot disagree about where they are. const VV_AI_KINDS = ['header', 'readme', 'manual', 'template', 'doc']; // Profiles — what each one is, and what it is allowed to see and do. One table, in one file, @@ -671,10 +672,9 @@ function vv_ai_token_prune(string $db): void { // Partner ledgers pulled by AI/ai_token_sync.sh into tmpfs. Same trick conf_sync.sh uses for // partner confs: the reader treats a missing file as "unknown", never as zero. // -// Mirrors AI_TOKEN_CACHE_DIR in load_config.sh. Hardcoded rather than read from conf because it -// is a derived path constant on the shell side too — neither end reads it from a conf file, so -// there is no single value to drift away from. -const VV_AI_TOKEN_CACHE_DIR = '/tmp/.cache/vv/ai'; +// The directory is AI_TOKEN_CACHE_DIR from master.conf, resolved in config.php. It used to be +// hardcoded here, on the reasoning that the shell side hardcoded it too and so there was no +// single value to drift away from — true when it was written, and no longer true. // Aggregates the file into today / last 7 days / all time, both overall and per host. // diff --git a/Plugin/unraid/include/config.php b/Plugin/unraid/include/config.php index ed6b9ed..1481422 100644 --- a/Plugin/unraid/include/config.php +++ b/Plugin/unraid/include/config.php @@ -57,7 +57,8 @@ // master.conf HOST / HOST_NAME — host identity // host*.conf HOST*_SSH_KEY — used for setup-state and conf push // ident.cfg timeZone — Unraid's own setting, adopted for the whole PHP layer -// VV_CACHE_DIR /tmp/vv_cache (tmpfs — RAM speed, cleared on reboot) +// master.conf VV_CACHE_ROOT and the cache paths derived from it — shared with +// load_config.sh, which reads the same keys from the same file // ═══════════════════════════════════════════════════════════════════════════════════════════════ // ── Timezone ────────────────────────────────────────────────────────────────── @@ -113,7 +114,45 @@ define('CUSTOM_SCRIPTS_DIR', $_vv_cfg['CUSTOM_SCRIPTS_DIR'] ?? '/boot/config/plu unset($_vv_cfg); define('VV_SETUP_STATE_FILE', STATE_DIR . '/varaverk_setup.db'); -define('VV_CACHE_DIR', '/tmp/vv_cache'); + +// ── Cache roots ─────────────────────────────────────────────────────────────── +// Read from master.conf so this layer and load_config.sh resolve the same paths from the same +// line. They used to be literals in three PHP files, restating what load_config.sh already said, +// because PHP cannot source bash — and the two sets were kept in agreement by hand. +// +// Deliberately NOT via vv_conf_vars(). That parses master.conf and this host's conf in full and +// calls vv_detect_host() on the way, and api/monitor.php's documented fast path loads this file +// and nothing else in order to reach vv_cache_read() on a cache hit. Making every request pay a +// full conf parse to learn a directory name would tax the exact path built to be cheap. One +// file, seven keys, no host detection. +// +// The fallbacks are the pre-consolidation paths, matching load_config.sh: an installation whose +// conf has not been through conf_upgrade yet keeps using what it is already using, rather than +// silently relocating its caches because a variable was missing. +$_vv_cache = []; +foreach (@file(CONF_DIR . '/master.conf') ?: [] as $_l) { + if (preg_match('/^\s*(VV_CACHE_ROOT|VV_CACHE_DIR|CONF_RAM_CACHE_DIR|ARR_CACHE_DIR' + . '|AI_TOKEN_CACHE_DIR|AI_JOB_DIR|DOCKER_JOB_DIR)\s*=\s*"?([^"#\r\n]+?)"?\s*(?:#.*)?$/', + $_l, $_m)) { + $_vv_cache[$_m[1]] = trim($_m[2]); + } +} +$_vv_root = $_vv_cache['VV_CACHE_ROOT'] ?? '/tmp/varaverk'; +// Resolves the one reference the conf actually uses. This is not a general bash expander and is +// not trying to be — vv_conf_vars() owns that, and every value here is one level deep by design. +$_vv_path = function (string $key, string $fallback) use ($_vv_cache, $_vv_root): string { + $v = trim($_vv_cache[$key] ?? ''); + if ($v === '') return $fallback; + return str_replace(['${VV_CACHE_ROOT}', '$VV_CACHE_ROOT'], $_vv_root, $v); +}; +define('VV_CACHE_ROOT', $_vv_root); +define('VV_CACHE_DIR', $_vv_path('VV_CACHE_DIR', '/tmp/vv_cache')); +define('VV_CONF_RAM_CACHE_DIR', $_vv_path('CONF_RAM_CACHE_DIR', '/tmp/.cache/vv/d')); +define('VV_ARR_CACHE_DIR', $_vv_path('ARR_CACHE_DIR', '/tmp/arr_cache')); +define('VV_AI_TOKEN_CACHE_DIR', $_vv_path('AI_TOKEN_CACHE_DIR', '/tmp/.cache/vv/ai')); +define('VV_AI_JOB_DIR', $_vv_path('AI_JOB_DIR', '/tmp/varaverk_ai_jobs')); +define('VV_JOB_DIR', $_vv_path('DOCKER_JOB_DIR', '/tmp/varaverk_dk_jobs')); +unset($_vv_cache, $_vv_root, $_vv_path, $_l, $_m); // Read the setup state file into a key=>value array. function vv_setup_state_read(): array { diff --git a/System_Essentials/conf_sync.sh b/System_Essentials/conf_sync.sh index 156da68..ccf085e 100755 --- a/System_Essentials/conf_sync.sh +++ b/System_Essentials/conf_sync.sh @@ -251,7 +251,7 @@ for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do fi if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would push ${MY_ID,,}.conf → $partner_host:/tmp/.cache/vv/d/" + warn "DRY RUN — would push ${MY_ID,,}.conf → $partner_host:$CACHE_DIR/" continue fi diff --git a/Watchdogs/System/conf_cache_watchdog.sh b/Watchdogs/System/conf_cache_watchdog.sh index 5d16611..a4e5b2f 100755 --- a/Watchdogs/System/conf_cache_watchdog.sh +++ b/Watchdogs/System/conf_cache_watchdog.sh @@ -146,7 +146,11 @@ require_partnership [[ "${CONF_SYNC_ENABLED:-true}" != "true" ]] && exit 0 [[ -z "${REMOTE_ID:-}" ]] && exit 0 -RAM_CACHE="/tmp/.cache/vv/d" +# The exported variable, not a literal. This was hardcoded while load_config.sh already exported +# CONF_RAM_CACHE_DIR, so the watchdog would have gone on watching an empty directory the moment +# that path moved — and reported the cache healthy because nothing was ever missing from a +# location nothing writes to. +RAM_CACHE="$CONF_RAM_CACHE_DIR" SAVE_DIR="${PERSISTENT_CONF_CACHE:-}" # SAVE_DIR is rm -rf'd below and is built from ${SCRIPTS_DIR}. If that is ever unset the diff --git a/load_config.sh b/load_config.sh index ae26cc1..f010fbd 100755 --- a/load_config.sh +++ b/load_config.sh @@ -76,8 +76,9 @@ # # It then defines the paths the rest of the ecosystem builds on: # -# CONF_RAM_CACHE_DIR /tmp/.cache/vv/d — tmpfs partner conf cache, cleared each reboot -# ARR_CACHE_DIR /tmp/arr_cache — tmpfs, restored from DATA_DIR on demand +# VV_CACHE_ROOT everything Varaverk keeps in RAM, one root, defined in master.conf +# CONF_RAM_CACHE_DIR $VV_CACHE_ROOT/conf — partner conf cache, chmod 700, snapshotted +# ARR_CACHE_DIR $VV_CACHE_ROOT/arr — restored from DATA_DIR on demand # # Everything else (STATE_DIR, DATA_DIR, thresholds, credentials) comes out of master.conf and # host*.conf, which this file sources rather than defines. @@ -130,6 +131,23 @@ fi source "$LOAD_CONFIG_DIR/Configurations/master.conf" +# ━━━ Cache roots ━━━ +# Resolved here, immediately after master.conf, rather than with the other derived paths further +# down — the partner-conf cache is read a few lines below to source HOST2_* vars, and that block +# used to carry its own copy of the path precisely because the variable did not exist yet at that +# point. Defining these the moment master.conf provides them removes the reason for the copy. +# +# The fallbacks are the pre-consolidation paths: an installation whose conf has not been through +# conf_upgrade keeps using what it is already using rather than silently relocating its caches. +# The PHP layer reads the same keys out of the same file, with the same fallbacks. + VV_CACHE_ROOT="${VV_CACHE_ROOT:-/tmp/varaverk}" + VV_CACHE_DIR="${VV_CACHE_DIR:-/tmp/vv_cache}" + CONF_RAM_CACHE_DIR="${CONF_RAM_CACHE_DIR:-/tmp/.cache/vv/d}" + ARR_CACHE_DIR="${ARR_CACHE_DIR:-/tmp/arr_cache}" + AI_TOKEN_CACHE_DIR="${AI_TOKEN_CACHE_DIR:-/tmp/.cache/vv/ai}" + AI_JOB_DIR="${AI_JOB_DIR:-/tmp/varaverk_ai_jobs}" + DOCKER_JOB_DIR="${DOCKER_JOB_DIR:-/tmp/varaverk_dk_jobs}" + # ━━━ Auto-discover and source all host*.conf files ━━━ # Sorted for consistent load order — HOST1 before HOST2 before HOST3 etc. # Each host conf extends the shared PROFILE_* arrays and adds host-specific vars. @@ -160,7 +178,7 @@ # (HOST2_*, HOST3_*, …) available without committing credentials to the git repo # or violating sparse checkout — partner confs live in RAM only, cleared on reboot. # Confs already loaded from disk are skipped — disk copy is authoritative. - _VV_CONF_CACHE="/tmp/.cache/vv/d" + _VV_CONF_CACHE="$CONF_RAM_CACHE_DIR" if [[ -d "$_VV_CONF_CACHE" ]]; then while IFS= read -r _conf; do [[ -f "$_conf" ]] || continue @@ -195,16 +213,11 @@ # re-deriving from SCRIPTS_DIR or hardcoding /var/log or /tmp paths inline. CONF_DIR="${SCRIPTS_DIR}/Configurations" LOG_DIR="/var/log/varaverk" - VV_CACHE_DIR="/tmp/vv_cache" - CONF_RAM_CACHE_DIR="/tmp/.cache/vv/d" # tmpfs — cleared every reboot, repopulated by conf_sync.sh - ARR_CACHE_DIR="/tmp/arr_cache" # tmpfs — cleared every reboot, restored from DATA_DIR backup by arr_cache_age_seconds() - # Partner AI ledgers pulled by AI/ai_token_sync.sh. A sibling of the conf cache rather than - # the same directory: that one holds credentials at chmod 700 and is snapshotted to /boot by - # conf_cache_save.sh. These are non-secret counters that are worthless when stale, so they - # must not be preserved across a reboot — losing them just means "not collected here" until - # the next sync, which is the honest answer anyway. - AI_TOKEN_CACHE_DIR="/tmp/.cache/vv/ai" # tmpfs — cleared every reboot, repopulated by ai_token_sync.sh - export CONF_DIR LOG_DIR VV_CACHE_DIR CONF_RAM_CACHE_DIR ARR_CACHE_DIR AI_TOKEN_CACHE_DIR + + # Cache roots were resolved right after master.conf was sourced, because the partner-conf + # cache is read before this point. Exported here with the rest. + export CONF_DIR LOG_DIR VV_CACHE_ROOT VV_CACHE_DIR CONF_RAM_CACHE_DIR ARR_CACHE_DIR \ + AI_TOKEN_CACHE_DIR AI_JOB_DIR DOCKER_JOB_DIR # ━━━ Cleanup ━━━ unset _conf _host_confs_loaded _adapter LOAD_CONFIG_DIR \ No newline at end of file