perf: cheaper ingest — no re-parse of live hooked transcripts, delta hook writes, lighter pollers

- Session sync leaves a transcript alone while hooks feed its session live
  (active, newest event a hook's, touched within DASHBOARD_SYNC_LIVE_HOOK_MS,
  default 5 min) and catches it up once quiet; a hook-created session (which
  importSession skips anyway) is only snapshotted, never parsed.
- Hook writer keeps a per-transcript mark (applied after commit): an
  unchanged TranscriptCache result skips token/compaction/API-error/turn
  writes, a grown one walks only the entries after the last one handled.
- Codex usage: one stamp query per pass instead of one row per journal;
  between full passes (first + every 5 min) only the newest day folders and
  journals written in the last hour are statted; fork parents resolved from
  the table or a full walk.
- Cursor/Antigravity polls back off to 60 s after idle passes
  (server/lib/backoff-poll.js); lane sync runs only while another dashboard
  shares the data dir (checked every 30 s).
This commit is contained in:
2026-10-16 10:20:09 +07:00
parent ecbbb4dd8f
commit 5cb70d702b
15 changed files with 734 additions and 51 deletions
+82
View File
@@ -150,3 +150,85 @@ describe("syncDefaultProjects", () => {
);
});
});
describe("syncDefaultProjects — sessions hooks are feeding", () => {
const SNAP_DIR = path.join(process.env.DASHBOARD_DATA_DIR, "transcripts");
const more = (sessionId, cwd, ts) => ({
type: "assistant",
cwd,
sessionId,
timestamp: ts,
message: {
model: "claude-opus-4-8",
content: [{ type: "text", text: "later" }],
usage: { input_tokens: 3, output_tokens: 2 },
},
});
const bumpMtime = (file) => {
const future = Date.now() / 1000 + 120;
fs.utimesSync(file, future, future);
};
const ageSession = (id) =>
dbModule.db
.prepare("UPDATE sessions SET updated_at = ? WHERE id = ?")
.run(new Date(Date.now() - 10 * 60_000).toISOString(), id);
it("defers a live hook-fed imported session, then catches it up once quiet", async () => {
const ID = "33333333-cccc-4ccc-8ccc-333333333333";
const cache = new Map();
const file = writeSession("-live", ID, fixtureLines(ID, "/live"));
await syncDefaultProjects(dbModule, { mtimeCache: cache });
assert.ok(dbModule.stmts.getSession.get(ID), "imported first");
// A hook arrives: its event is the newest, and it touches the session.
dbModule.stmts.insertEvent.run(ID, `${ID}-main`, "PreToolUse", "Bash", "hook", "{}");
dbModule.stmts.touchSession.run(ID);
writeSession(
"-live",
ID,
fixtureLines(ID, "/live", [more(ID, "/live", "2026-04-18T12:09:00.000Z")])
);
bumpMtime(file);
const live = await syncDefaultProjects(dbModule, { mtimeCache: cache });
assert.equal(
live.changed.find((c) => c.sessionId === ID),
undefined,
"deferred while live"
);
assert.equal(cache.get(file), -1, "kept in the cache so the watcher ignores its appends");
ageSession(ID); // the hooks went quiet
const quiet = await syncDefaultProjects(dbModule, { mtimeCache: cache });
assert.ok(
quiet.changed.find((c) => c.sessionId === ID && !c.isNew),
"caught up once quiet"
);
assert.notEqual(cache.get(file), -1);
});
it("never re-parses a hook-created session, only refreshes its snapshot once quiet", async () => {
const ID = "44444444-dddd-4ddd-8ddd-444444444444";
dbModule.stmts.insertSession.run(ID, "hooked", "active", "/hooked", null, null);
dbModule.stmts.insertEvent.run(ID, null, "SessionStart", null, "hook", "{}");
const file = writeSession("-hooked", ID, fixtureLines(ID, "/hooked"));
bumpMtime(file);
const snap = path.join(SNAP_DIR, `${ID}.jsonl`);
const cache = new Map();
await syncDefaultProjects(dbModule, { mtimeCache: cache });
assert.equal(fs.existsSync(snap), false, "no snapshot copy while hooks are live");
ageSession(ID);
const before = dbModule.db.prepare("SELECT COUNT(*) AS n FROM events WHERE session_id = ?");
const n0 = before.get(ID).n;
const r = await syncDefaultProjects(dbModule, { mtimeCache: cache });
assert.equal(
r.changed.find((c) => c.sessionId === ID),
undefined,
"nothing imported"
);
assert.equal(before.get(ID).n, n0, "no events written for a hook-created session");
assert.equal(fs.existsSync(snap), true, "snapshot taken once quiet");
});
});