perf: cheaper ingest — no re-parse of live hooked transcripts, delta hook writes, lighter pollers

- Session sync leaves a transcript alone while hooks feed its session live (active, newest event a hook's, touched within DASHBOARD_SYNC_LIVE_HOOK_MS, default 5 min) and catches it up once quiet; a hook-created session (which importSession skips anyway) is only snapshotted, never parsed. - Hook writer keeps a per-transcript mark (applied after commit): an unchanged TranscriptCache result skips token/compaction/API-error/turn writes, a grown one walks only the entries after the last one handled. - Codex usage: one stamp query per pass instead of one row per journal; between full passes (first + every 5 min) only the newest day folders and journals written in the last hour are statted; fork parents resolved from the table or a full walk. - Cursor/Antigravity polls back off to 60 s after idle passes (server/lib/backoff-poll.js); lane sync runs only while another dashboard shares the data dir (checked every 30 s).
2026-10-16 10:20:09 +07:00
parent ecbbb4dd8f
commit 5cb70d702b
15 changed files with 734 additions and 51 deletions
@@ -536,3 +536,78 @@ describe("service tier", () => {
    ]);
  });
 });
+
+describe("ingestCodexUsage — recent pass", () => {
+  const NOW = Date.parse("2026-09-29T03:00:00Z");
+  function writeAt(day, id, lines, mtime) {
+    const dir = path.join(process.env.CODEX_HOME, "sessions", ...day.split("/"));
+    fs.mkdirSync(dir, { recursive: true });
+    const file = path.join(dir, `rollout-${day.replace(/\//g, "-")}T01-00-00-${id}.jsonl`);
+    fs.writeFileSync(file, jsonl(lines));
+    fs.utimesSync(file, mtime, mtime);
+    return file;
+  }
+
+  it("stats only the newest day folders and the journals still growing", async () => {
+    const quiet = writeAt(
+      "2026/09/01",
+      "old1",
+      [meta("old1"), ctx("gpt-a"), tc([10, 0, 1], [10, 0, 1])],
+      new Date(NOW - 20 * 86_400_000)
+    );
+    const live = writeAt(
+      "2026/09/02",
+      "old2",
+      [meta("old2"), ctx("gpt-a"), tc([10, 0, 1], [10, 0, 1])],
+      new Date(NOW - 60_000)
+    );
+    writeAt("2026/09/27", "a", [meta("a"), ctx("gpt-a")], new Date(NOW - 86_400_000));
+    writeAt("2026/09/28", "b", [meta("b"), ctx("gpt-a")], new Date(NOW - 86_400_000));
+    await ingestCodexUsage({ now: NOW }); // the full first pass
+
+    // Both old journals grow; only the one written within the hour is looked at.
+    fs.appendFileSync(quiet, jsonl([tc([30, 0, 3], [20, 0, 2])]));
+    fs.appendFileSync(live, jsonl([tc([30, 0, 3], [20, 0, 2])]));
+    fs.utimesSync(quiet, new Date(NOW - 1000), new Date(NOW - 1000));
+    fs.utimesSync(live, new Date(NOW - 1000), new Date(NOW - 1000));
+    // A new journal in today's folder is found too.
+    writeAt(
+      "2026/09/29",
+      "new1",
+      [meta("new1"), ctx("gpt-a"), tc([5, 0, 1], [5, 0, 1])],
+      new Date(NOW - 1000)
+    );
+    await ingestCodexUsage({ now: NOW, recent: true });
+    assert.deepEqual(usage("old2"), [{ model: "gpt-a", i: 30, r: 0, o: 3 }]);
+    assert.deepEqual(usage("old1"), [{ model: "gpt-a", i: 10, r: 0, o: 1 }], "left to a full pass");
+    assert.deepEqual(usage("new1"), [{ model: "gpt-a", i: 5, r: 0, o: 1 }]);
+
+    await ingestCodexUsage({ now: NOW }); // the next full pass catches it up
+    assert.deepEqual(usage("old1"), [{ model: "gpt-a", i: 30, r: 0, o: 3 }]);
+  });
+
+  it("finds a fork's parent in an older folder on a recent pass", async () => {
+    const parent = [
+      meta("p9"),
+      ctx("gpt-a"),
+      tc([100, 0, 10], [100, 0, 10]),
+      tc([300, 0, 30], [200, 0, 20]),
+    ];
+    writeAt("2026/08/01", "p9", parent, new Date(NOW - 50 * 86_400_000));
+    writeAt("2026/09/28", "x", [meta("x")], new Date(NOW - 86_400_000));
+    await ingestCodexUsage({ now: NOW });
+    writeAt(
+      "2026/09/29",
+      "f9",
+      [
+        meta("f9", { forked_from_id: "p9" }),
+        meta("p9"),
+        ...parent.slice(1),
+        tc([350, 0, 35], [50, 0, 5]),
+      ],
+      new Date(NOW - 1000)
+    );
+    await ingestCodexUsage({ now: NOW, recent: true });
+    assert.deepEqual(usage("f9"), [{ model: "gpt-a", i: 50, r: 0, o: 5 }]);
+  });
+});