Varaverk/Plugin/unraid/include/ai_memory_learn.php

<?php
// ═══════════════════════════════════════════════════════════════════════════════════════════════
// ai_memory_learn.php — proposed memory: the assistant suggests, the operator keeps
// ═══════════════════════════════════════════════════════════════════════════════════════════════
//
// PURPOSE
//   Lets the assistant offer facts it thinks are worth remembering, without letting it write to
//   the prompt on its own. A proposal is filed; the operator accepts or dismisses it; accepted
//   text lands in the learned memory slot, which the prompt explicitly ranks BELOW retrieval.
//
// WHY IT IS A PROPOSAL AND NOT A WRITE
//   Memory is injected into every future prompt. A model that writes its own memory writes its
//   own mistakes, and then reads them back as established fact — growing more confident on each
//   turn while the actual source code says otherwise. The cost of a bad proposal has to be one
//   dismissal, not a permanently poisoned prompt. This is the same two-gate shape the repair
//   system uses, for the same reason: whether something should be remembered is intent, and a
//   model cannot prove intent.
//
// WHY DEDUP IS NOT THE MODEL'S JOB
//   "Do I already know this" is a semantic comparison, and a 14B at IQ4_XS is confidently wrong
//   at it often enough to matter — with every miss costing budget permanently. So dedup here is
//   deterministic: normalise, then reject on exact match or containment in either direction
//   against assisted memory, learned memory, and everything previously dismissed. It will let
//   through a reworded duplicate; it will never silently drop something new, and that is the
//   right way round for a store the operator reviews anyway.
//
// GATES
//   AI_MEMORY_LEARN_ENABLED       false — nothing is proposed, and the prompt gains nothing
//   AI_MEMORY_LEARN_AUTO_ACCEPT   false — accepted writes happen only when the operator says so
//   The second cannot outrank the first: auto-accept with proposing off does nothing at all.
//
// STORE
//   data/ai/mem_proposals/<id>.json — one file per proposal, mirroring the findings store.
//   States: open | accepted | dismissed. Dismissed rows are KEPT, because they are what stops
//   the same suggestion arriving again every night.
// ═══════════════════════════════════════════════════════════════════════════════════════════════

require_once __DIR__ . '/ai.php';

function vv_ai_mem_learn_enabled(): bool {
    $v = vv_conf_vars();
    return strtolower(trim($v['AI_MEMORY_LEARN_ENABLED'] ?? 'false')) === 'true';
}

// Cannot outrank its parent — the same rule autofix follows against AI_REPAIR_ENABLED.
function vv_ai_mem_learn_auto(): bool {
    if (!vv_ai_mem_learn_enabled()) return false;
    $v = vv_conf_vars();
    return strtolower(trim($v['AI_MEMORY_LEARN_AUTO_ACCEPT'] ?? 'false')) === 'true';
}

function vv_ai_mem_dir(): string {
    $d = AI_DATA_DIR . '/mem_proposals';
    if (!is_dir($d)) @mkdir($d, 0755, true);
    return $d;
}

// Aggressive on purpose: case, punctuation and filler collapse away so that "Emby's ffprobe is
// broken." and "emby ffprobe is broken" compare equal. Only used for comparison — never stored.
function vv_ai_mem_norm(string $s): string {
    $s = mb_strtolower(trim($s));
    $s = preg_replace('/^[\s\-\*\d\.\)]+/u', '', $s);   // list markers
    $s = preg_replace('/[^a-z0-9]+/u', ' ', $s);
    // Stray single LETTERS go too. "Emby's" becomes "emby s", and that orphaned s is enough to
    // stop "emby bundled ffprobe is broken" matching "emby's bundled ffprobe is broken" — a
    // reworded duplicate slipping through on an apostrophe. Digits are kept, so a PCIe address
    // like 0000:02:02.0 still normalises to something that compares against itself.
    $s = preg_replace('/\b[a-z]\b/u', ' ', $s);
    return trim(preg_replace('/\s+/', ' ', $s));
}

// Everything the operator has already seen, in normalised form: both memory slots line by line,
// plus every proposal ever filed. Dismissed ones count — re-offering something that was turned
// down is the fastest way to make the whole feature feel broken.
function vv_ai_mem_known(): array {
    $known = [];

    foreach (['assisted', 'learned'] as $kind) {
        $p = vv_ai_memory_path($kind);
        if (!file_exists($p)) continue;
        foreach (preg_split('/\R/', (string)@file_get_contents($p)) as $line) {
            $n = vv_ai_mem_norm($line);
            if ($n !== '' && mb_strlen($n) > 8) $known[] = $n;
        }
    }

    foreach (glob(vv_ai_mem_dir() . '/*.json') ?: [] as $f) {
        $d = json_decode((string)@file_get_contents($f), true);
        if (!is_array($d)) continue;
        $n = vv_ai_mem_norm((string)($d['text'] ?? ''));
        if ($n !== '') $known[] = $n;
    }

    return $known;
}

// Containment both ways, because a proposal is usually a longer or shorter phrasing of something
// already held rather than a character-identical repeat.
function vv_ai_mem_is_dup(string $text, ?array $known = null): bool {
    $n = vv_ai_mem_norm($text);
    if ($n === '') return true;
    foreach ($known ?? vv_ai_mem_known() as $k) {
        if ($k === $n) return true;
        if (mb_strlen($k) > 12 && mb_strpos($n, $k) !== false) return true;
        if (mb_strlen($n) > 12 && mb_strpos($k, $n) !== false) return true;
    }
    return false;
}

// A proposal is a single sentence. Anything longer is the model summarising the conversation
// rather than stating a fact, and it would eat the budget it is competing for.
const VV_AI_MEM_MAX_LINE = 220;

// Where a setting lives in the UI is documented, and documentation does not belong in memory —
// memory is for what the documents cannot tell you. The UI map made this a real problem rather
// than a theoretical one: it describes 447 settings, so without this the assistant will propose
// "X is on the Settings tab" over and over, each one costing a dismissal.
//
// Gated on WHICH document the candidate is nearest to, not on how near. A plain score threshold
// was tried first and does not separate these: a genuine fact about rsync scores 0.78 against
// rsync.sh because it is about rsync, higher than the junk proposal that started this at 0.75.
// Similarity measures topic, not novelty. But the junk lands on the UI map and nothing worth
// remembering ever has — that is a difference in kind, and kind is what the index records.
const VV_AI_MEM_UI_NEAR = 0.62;

function vv_ai_mem_is_ui_fact(string $text): bool {
    // Retrieval needs the index and the embedder. If either is unavailable the answer is "not
    // provably documented", which files the proposal — the operator can still dismiss it, and
    // failing closed here would silently stop learning whenever Ollama was restarting.
    $r = @vv_ai_retrieve($text, '', '', 1);
    if (empty($r['ok']) || empty($r['results'])) return false;
    $top = $r['results'][0];
    return ($top['kind'] ?? '') === 'ui' && (float) ($top['score'] ?? 0) >= VV_AI_MEM_UI_NEAR;
}

function vv_ai_mem_propose(string $text, array $meta = []): array {
    if (!vv_ai_mem_learn_enabled()) return ['ok' => false, 'error' => 'learning disabled'];

    $text = trim(preg_replace('/\s+/', ' ', $text));
    if ($text === '')                             return ['ok' => false, 'error' => 'empty'];
    if (mb_strlen($text) > VV_AI_MEM_MAX_LINE)    return ['ok' => false, 'error' => 'too long'];
    if (vv_ai_mem_is_dup($text))                  return ['ok' => false, 'error' => 'duplicate'];
    if (vv_ai_mem_is_ui_fact($text))              return ['ok' => false, 'error' => 'already in the UI map'];

    $id  = substr(hash('sha256', $text . microtime(true)), 0, 12);
    $row = [
        'id'      => $id,
        'text'    => $text,
        'state'   => 'open',
        'created' => time(),
        'profile' => (string)($meta['profile'] ?? ''),
        // The question that produced it, trimmed. Without this a line read three weeks later has
        // no way to be judged — the same reason findings carry their evidence.
        'asked'   => mb_substr(trim((string)($meta['asked'] ?? '')), 0, 160),
    ];

    if (vv_ai_mem_learn_auto()) {
        $r = vv_ai_mem_append($text);
        $row['state']    = $r['ok'] ? 'accepted' : 'open';
        $row['auto']     = true;
        $row['accepted'] = $r['ok'] ? time() : null;
        if (!$r['ok']) $row['error'] = $r['error'];
    }

    if (!vv_ai_mem_write_row(vv_ai_mem_dir() . "/$id.json", $row)) {
        return ['ok' => false, 'error' => 'could not file the proposal'];
    }
    return ['ok' => true, 'id' => $id, 'state' => $row['state']];
}

// Appends one line to the learned slot, respecting its own ceiling. Refuses rather than trims:
// silently dropping half a fact is worse than declining to add it.
function vv_ai_mem_append(string $text): array {
    $p   = vv_ai_memory_path('learned');
    $max = vv_ai_memory_learned_max();
    if ($max <= 0) return ['ok' => false, 'error' => 'learned memory is disabled (cap is 0)'];

    $cur = file_exists($p) ? rtrim((string)@file_get_contents($p)) : '';
    if ($cur === '') {
        $cur = "# Learned — proposed by the assistant, kept by the operator.\n"
             . "# Hints only: anything retrieved from the repo outranks these.\n";
    }
    $line = '- ' . $text . '  (' . date('Y-m-d') . ')';
    $next = $cur . "\n" . $line . "\n";

    if (mb_strlen($next) > $max) {
        return ['ok' => false,
                'error' => 'learned memory is full (' . mb_strlen($cur) . '/' . $max
                         . ' chars) — remove a line before adding another'];
    }
    return vv_ai_memory_write($next, 'learned');
}

// Takes one line back out of the learned slot. The counterpart to vv_ai_mem_append(), and the
// reason a decided proposal can be changed at all: without this, "accepted" was a one-way door
// and the only way out was editing mem_learned.md by hand. An accepted line joins every future
// prompt, and a fact that is true today — HOST2 is offline — becomes actively wrong the day it
// stops being true, with nothing to expire it.
//
// Matched on the line this file wrote: "- <text>  (<date>)". A line the operator has since edited
// by hand will not match, and that is reported rather than swallowed — silently succeeding while
// the text stays in every prompt is the one outcome worse than failing.
function vv_ai_mem_remove(string $text): array {
    $p = vv_ai_memory_path('learned');
    if (!file_exists($p)) return ['ok' => false, 'error' => 'learned memory file does not exist'];

    $cur   = (string) @file_get_contents($p);
    $lines = preg_split('/\R/', $cur);
    $want  = vv_ai_mem_norm($text);
    $out   = [];
    $hit   = false;

    foreach ($lines as $line) {
        // Compare the line's own text, stripped of the bullet and the trailing date stamp, so a
        // change in date format does not strand the entry.
        $bare = preg_replace('/^\s*-\s*/', '', $line);
        $bare = preg_replace('/\s*\(\d{4}-\d{2}-\d{2}\)\s*$/', '', $bare);
        if (!$hit && $bare !== '' && vv_ai_mem_norm($bare) === $want) { $hit = true; continue; }
        $out[] = $line;
    }
    if (!$hit) return ['ok' => false, 'error' => 'that line is not in learned memory — it may have been edited by hand'];

    $next = rtrim(implode("\n", $out)) . "\n";
    // Only the header left means nothing is being remembered; write it back empty rather than
    // leaving a file that looks populated.
    if (trim(preg_replace('/^#.*$/m', '', $next)) === '') $next = '';
    return vv_ai_memory_write($next, 'learned');
}

function vv_ai_mem_list(string $state = ''): array {
    $out = [];
    foreach (glob(vv_ai_mem_dir() . '/*.json') ?: [] as $f) {
        $d = json_decode((string)@file_get_contents($f), true);
        if (!is_array($d)) continue;
        if ($state !== '' && ($d['state'] ?? '') !== $state) continue;
        $out[] = $d;
    }
    usort($out, fn($a, $b) => ($b['created'] ?? 0) <=> ($a['created'] ?? 0));
    return $out;
}

// Temp and rename, so a concurrent list() never decodes half a record — a torn read here is
// indistinguishable from a corrupt proposal and the row silently vanishes from the card.
function vv_ai_mem_write_row(string $f, array $row): bool {
    $tmp = $f . '.tmp';
    if (@file_put_contents($tmp, json_encode($row, JSON_PRETTY_PRINT)) === false) {
        @unlink($tmp);
        return false;
    }
    if (!@rename($tmp, $f)) { @unlink($tmp); return false; }
    return true;
}

// One lock for every decision, not one per proposal. Accept is a read-modify-write across two
// files — this proposal and the learned slot — and two of them interleaving is how the same line
// lands in memory twice, or how one of two accepted lines is lost to a temp-and-rename that began
// before the other finished. The card arms and disables the button, so this only has to hold
// against a second tab, a double submit or an impatient reload; that is exactly when it matters,
// because none of those are visible from the one that is about to lose.
function vv_ai_mem_action(string $id, string $act): array {
    // open      → accept | dismiss      the original decision
    // accepted  → retract               takes the line back out of every future prompt
    // dismissed → keep                  changes your mind, puts it back
    // decided   → forget                drops the record entirely
    //
    // A decision used to be final, which made "accepted" a one-way door into the prompt. The
    // transitions below are the way back out; each is checked against the state it is legal from,
    // so a stale tab cannot retract something that was already forgotten.
    $legal = ['accept' => 'open', 'dismiss' => 'open',
              'retract' => 'accepted', 'keep' => 'dismissed', 'forget' => '*'];
    if (!preg_match('/^[0-9a-f]{12}$/', $id)) return ['ok' => false, 'error' => 'bad id'];
    if (!isset($legal[$act]))                 return ['ok' => false, 'error' => 'unknown action'];

    $f = vv_ai_mem_dir() . "/$id.json";
    if (!file_exists($f)) return ['ok' => false, 'error' => 'no such proposal'];

    $lock = @fopen(vv_ai_mem_dir() . '/.lock', 'c');
    if ($lock === false || !flock($lock, LOCK_EX)) {
        if ($lock !== false) fclose($lock);
        return ['ok' => false, 'error' => 'could not lock the proposal store'];
    }

    try {
        // Re-read under the lock. Whatever the card was showing when it was clicked is not
        // evidence of anything — the decision may already have been made in another tab.
        $d = json_decode((string)@file_get_contents($f), true);
        if (!is_array($d)) return ['ok' => false, 'error' => 'unreadable proposal'];

        // A tab left open overnight must not act on a choice the store has already moved past.
        $state = (string) ($d['state'] ?? '');
        if ($legal[$act] !== '*' && $state !== $legal[$act]) {
            return ['ok' => false, 'error' => 'this is ' . ($state ?: 'in no state')
                                            . ', so it cannot be ' . $act . 'ed'];
        }

        if ($act === 'accept' || $act === 'keep') {
            $r = vv_ai_mem_append((string)$d['text']);
            if (!$r['ok']) return $r;
            $d['state']    = 'accepted';
            $d['accepted'] = time();
            unset($d['closed']);
        } elseif ($act === 'dismiss') {
            $d['state']  = 'dismissed';
            $d['closed'] = time();
        } elseif ($act === 'retract') {
            // The line leaves memory first. If that fails the proposal keeps saying "accepted",
            // which is the truth — the text is still in every prompt.
            $r = vv_ai_mem_remove((string)$d['text']);
            if (!$r['ok']) return $r;
            $d['state']  = 'dismissed';
            $d['closed'] = time();
            unset($d['accepted']);
        } elseif ($act === 'forget') {
            // Retract first when it is live, or the record vanishes while its text stays in the
            // prompt with nothing left pointing at it.
            if ($state === 'accepted') {
                $r = vv_ai_mem_remove((string)$d['text']);
                if (!$r['ok']) return $r;
            }
            // Dropping the record also drops what stops it being proposed again — dedup checks
            // every past proposal including dismissed ones. That is the point of forgetting
            // rather than dismissing, and it is why the button says so.
            if (!@unlink($f)) return ['ok' => false, 'error' => 'could not remove the proposal'];
            return ['ok' => true, 'state' => 'forgotten'];
        }

        if (!vv_ai_mem_write_row($f, $d)) {
            return ['ok' => false, 'error' => 'could not write the proposal'];
        }
        return ['ok' => true, 'state' => $d['state']];
    } finally {
        flock($lock, LOCK_UN);
        fclose($lock);
    }
}