fix(memory-core): filter REM dreaming candidates to light-staged entries (#86302)

* fix(memory-core): filter REM dreaming candidates to light-staged entries

REM dreaming re-ingested the full short-term recall store independently,
ignoring which entries were staged by the light sleep phase. Because the
confidence formula heavily weights accumulated averageScore (45%) and
recallStrength (25%), old high-recall entries permanently dominated
freshly staged candidates. The intended light→REM→deep pipeline was
broken: light correctly staged current material, but REM selected a
different set entirely, so lightHits never paired with remHits for deep
ranking.

Fix: in runRemDreaming(), read the phase-signals store for keys with
lightHits > 0 and filter entries to that set before passing to
previewRemDreaming(). When no light-staged keys exist (light disabled
or first run), fall back to the full entry set for backward
compatibility.

Added readLightStagedKeys() to short-term-promotion.ts as a clean
export for reading the light-staged key set from the phase signal store.

Closes #86249

Signed-off-by: Sebastien Tardif <sebtardif@ncf.ca>

* fix(memory-core): keep REM staging pending

* fix(memory-core): mark REM-considered staged entries

---------

Signed-off-by: Sebastien Tardif <sebtardif@ncf.ca>
Co-authored-by: Peter Steinberger <steipete@gmail.com>

extensions/memory-core/src/dreaming-phases.ts

@@ -28,8 +28,10 @@ import {
 import { asRecord, formatErrorMessage, normalizeTrimmedString } from "./dreaming-shared.js";
 import {
   filterLiveShortTermRecallEntries,
+  readLightStagedKeys,
   readShortTermRecallEntries,
   recordDreamingPhaseSignals,
+  recordRemConsideredPhaseSignals,
   recordShortTermRecalls,
   type ShortTermRecallEntry,
 } from "./short-term-promotion.js";
@@ -1723,7 +1725,7 @@ async function runRemDreaming(params: {
     nowMs,
     timezone: params.config.timezone,
   });
-  const entries = await filterLiveShortTermRecallEntries({
+  const allEntries = await filterLiveShortTermRecallEntries({
     workspaceDir: params.workspaceDir,
     entries: filterRecallEntriesWithinLookback({
       entries: await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs }),
@@ -1731,6 +1733,15 @@ async function runRemDreaming(params: {
       lookbackDays: params.config.lookbackDays,
     }),
   });
+  // Prefer entries staged by light sleep so REM synthesises from the
+  // sequential light→REM pipeline instead of rescanning the full store.
+  const lightKeys = await readLightStagedKeys({
+    workspaceDir: params.workspaceDir,
+    nowMs,
+  });
+  const stagedEntries =
+    lightKeys.size > 0 ? allEntries.filter((entry) => lightKeys.has(entry.key)) : [];
+  const entries = stagedEntries.length > 0 ? stagedEntries : allEntries;
   const preview = previewRemDreaming({
     entries,
     limit: params.config.limit,
@@ -1744,6 +1755,13 @@ async function runRemDreaming(params: {
     timezone: params.config.timezone,
     storage: params.config.storage,
   });
+  if (stagedEntries.length > 0) {
+    await recordRemConsideredPhaseSignals({
+      workspaceDir: params.workspaceDir,
+      keys: stagedEntries.map((entry) => entry.key),
+      nowMs,
+    });
+  }
   await recordDreamingPhaseSignals({
     workspaceDir: params.workspaceDir,
     phase: "rem",

extensions/memory-core/src/short-term-promotion.test.ts

@@ -14,7 +14,9 @@ import {
   recordGroundedShortTermCandidates,
   rankShortTermPromotionCandidates,
   recordDreamingPhaseSignals,
+  recordRemConsideredPhaseSignals,
   recordShortTermRecalls,
+  readLightStagedKeys,
   removeGroundedShortTermCandidates,
   repairShortTermPromotionArtifacts,
   resolveShortTermRecallLockPath,
@@ -492,6 +494,83 @@ describe("short-term promotion", () => {
     });
   });
 
+  it("reads only light-staged keys that have not already gone through REM", async () => {
+    await withTempWorkspace(async (workspaceDir) => {
+      const nowMs = Date.parse("2026-04-05T10:00:00.000Z");
+      await recordShortTermRecalls({
+        workspaceDir,
+        query: "phase pipeline",
+        nowMs,
+        results: [
+          {
+            path: "memory/2026-04-01.md",
+            startLine: 1,
+            endLine: 1,
+            score: 0.9,
+            snippet: "Move backups to S3 Glacier.",
+            source: "memory",
+          },
+          {
+            path: "memory/2026-04-02.md",
+            startLine: 1,
+            endLine: 1,
+            score: 0.91,
+            snippet: "Document the Ollama setup.",
+            source: "memory",
+          },
+        ],
+      });
+      const ranked = await rankShortTermPromotionCandidates({
+        workspaceDir,
+        minScore: 0,
+        minRecallCount: 0,
+        minUniqueQueries: 0,
+        nowMs,
+      });
+      const staleKey = requireCandidateKey(
+        ranked.find((entry) => entry.path === "memory/2026-04-01.md"),
+        "stale candidate",
+      );
+      const pendingKey = requireCandidateKey(
+        ranked.find((entry) => entry.path === "memory/2026-04-02.md"),
+        "pending candidate",
+      );
+
+      await recordDreamingPhaseSignals({
+        workspaceDir,
+        phase: "light",
+        keys: [staleKey],
+        nowMs: nowMs - 60_000,
+      });
+      await recordDreamingPhaseSignals({
+        workspaceDir,
+        phase: "rem",
+        keys: [staleKey],
+        nowMs,
+      });
+      await recordDreamingPhaseSignals({
+        workspaceDir,
+        phase: "light",
+        keys: [pendingKey],
+        nowMs: nowMs + 60_000,
+      });
+
+      await expect(readLightStagedKeys({ workspaceDir, nowMs: nowMs + 120_000 })).resolves.toEqual(
+        new Set([pendingKey]),
+      );
+
+      await recordRemConsideredPhaseSignals({
+        workspaceDir,
+        keys: [pendingKey],
+        nowMs: nowMs + 180_000,
+      });
+
+      await expect(readLightStagedKeys({ workspaceDir, nowMs: nowMs + 240_000 })).resolves.toEqual(
+        new Set(),
+      );
+    });
+  });
+
   it("lets grounded durable evidence satisfy default deep thresholds", async () => {
     await withTempWorkspace(async (workspaceDir) => {
       await writeDailyMemoryNote(workspaceDir, "2026-04-03", [

extensions/memory-core/src/short-term-promotion.ts

@@ -96,6 +96,7 @@ type ShortTermPhaseSignalEntry = {
   remHits: number;
   lastLightAt?: string;
   lastRemAt?: string;
+  lastRemConsideredAt?: string;
 };
 
 type ShortTermPhaseSignalStore = {
@@ -831,12 +832,17 @@ function normalizePhaseSignalStore(raw: unknown, nowIso: string): ShortTermPhase
       typeof entry.lastRemAt === "string" && entry.lastRemAt.trim().length > 0
         ? entry.lastRemAt
         : undefined;
+    const lastRemConsideredAt =
+      typeof entry.lastRemConsideredAt === "string" && entry.lastRemConsideredAt.trim().length > 0
+        ? entry.lastRemConsideredAt
+        : undefined;
     entries[key] = {
       key,
       lightHits,
       remHits,
       ...(lastLightAt ? { lastLightAt } : {}),
       ...(lastRemAt ? { lastRemAt } : {}),
+      ...(lastRemConsideredAt ? { lastRemConsideredAt } : {}),
     };
   }
   return {
@@ -1222,6 +1228,86 @@ export async function recordDreamingPhaseSignals(params: {
   });
 }
 
+export async function recordRemConsideredPhaseSignals(params: {
+  workspaceDir?: string;
+  keys: string[];
+  nowMs?: number;
+}): Promise<void> {
+  const workspaceDir = params.workspaceDir?.trim();
+  if (!workspaceDir) {
+    return;
+  }
+  const keys = [...new Set(params.keys.map((key) => key.trim()).filter(Boolean))];
+  if (keys.length === 0) {
+    return;
+  }
+  const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now();
+  const nowIso = new Date(nowMs).toISOString();
+
+  await withShortTermLock(workspaceDir, async () => {
+    const [store, phaseSignals] = await Promise.all([
+      readStore(workspaceDir, nowIso),
+      readPhaseSignalStore(workspaceDir, nowIso),
+    ]);
+    const knownKeys = new Set(Object.keys(store.entries));
+
+    for (const key of keys) {
+      if (!knownKeys.has(key)) {
+        continue;
+      }
+      const entry = phaseSignals.entries[key] ?? {
+        key,
+        lightHits: 0,
+        remHits: 0,
+      };
+      entry.lastRemConsideredAt = nowIso;
+      phaseSignals.entries[key] = entry;
+    }
+
+    for (const [key, entry] of Object.entries(phaseSignals.entries)) {
+      if (!knownKeys.has(key) || (entry.lightHits <= 0 && entry.remHits <= 0)) {
+        delete phaseSignals.entries[key];
+      }
+    }
+
+    phaseSignals.updatedAt = nowIso;
+    await writePhaseSignalStore(workspaceDir, phaseSignals);
+  });
+}
+
+export async function readLightStagedKeys(params: {
+  workspaceDir: string;
+  nowMs?: number;
+}): Promise<Set<string>> {
+  const workspaceDir = params.workspaceDir?.trim();
+  if (!workspaceDir) {
+    return new Set();
+  }
+  const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now();
+  const nowIso = new Date(nowMs).toISOString();
+  const store = await readPhaseSignalStore(workspaceDir, nowIso);
+  const keys = new Set<string>();
+  for (const [key, entry] of Object.entries(store.entries)) {
+    if (entry.lightHits <= 0) {
+      continue;
+    }
+    const lastLightMs = Date.parse(entry.lastLightAt ?? "");
+    const lastRemMs = Date.parse(entry.lastRemAt ?? "");
+    const lastRemConsideredMs = Date.parse(entry.lastRemConsideredAt ?? "");
+    const lastConsumedMs = Math.max(
+      Number.isFinite(lastRemMs) ? lastRemMs : Number.NEGATIVE_INFINITY,
+      Number.isFinite(lastRemConsideredMs) ? lastRemConsideredMs : Number.NEGATIVE_INFINITY,
+    );
+    const hasPendingLightSignal = Number.isFinite(lastLightMs)
+      ? lastLightMs > lastConsumedMs
+      : !entry.lastRemAt;
+    if (hasPendingLightSignal) {
+      keys.add(key);
+    }
+  }
+  return keys;
+}
+
 export async function rankShortTermPromotionCandidates(
   options: RankShortTermPromotionOptions,
 ): Promise<PromotionCandidate[]> {