perf: cheaper ingest — no re-parse of live hooked transcripts, delta hook writes, lighter pollers

- Session sync leaves a transcript alone while hooks feed its session live
  (active, newest event a hook's, touched within DASHBOARD_SYNC_LIVE_HOOK_MS,
  default 5 min) and catches it up once quiet; a hook-created session (which
  importSession skips anyway) is only snapshotted, never parsed.
- Hook writer keeps a per-transcript mark (applied after commit): an
  unchanged TranscriptCache result skips token/compaction/API-error/turn
  writes, a grown one walks only the entries after the last one handled.
- Codex usage: one stamp query per pass instead of one row per journal;
  between full passes (first + every 5 min) only the newest day folders and
  journals written in the last hour are statted; fork parents resolved from
  the table or a full walk.
- Cursor/Antigravity polls back off to 60 s after idle passes
  (server/lib/backoff-poll.js); lane sync runs only while another dashboard
  shares the data dir (checked every 30 s).
This commit is contained in:
2026-10-16 10:20:09 +07:00
parent ecbbb4dd8f
commit 5cb70d702b
15 changed files with 734 additions and 51 deletions
+60
View File
@@ -1937,6 +1937,47 @@ async function ingestWorkflowsForBatch(dbModule, sessions) {
}
}
// Session sync: a session counts as fed LIVE by hooks while it is active, its
// newest event came from a hook (not this importer), and a hook touched it
// within this window. Such a transcript is left to the hooks until it goes
// quiet. DASHBOARD_SYNC_LIVE_HOOK_MS=0 turns the deferral off.
const LIVE_HOOK_MS = (() => {
const n = Number(process.env.DASHBOARD_SYNC_LIVE_HOOK_MS ?? 5 * 60_000);
return Number.isFinite(n) && n >= 0 ? n : 5 * 60_000;
})();
// mtimeCache value for a deferred file: present (so the fs watcher ignores its
// appends) yet never equal to a real mtime (so every sweep re-checks it).
const DEFERRED_MTIME = -1;
const IMPORTED_DATA_RE = /"imported"\s*:\s*true/;
const latestEventStmts = new WeakMap();
function rowIsImported(row) {
try {
return !!(row.metadata && JSON.parse(row.metadata).imported);
} catch {
return false;
}
}
function isHookLive(dbModule, row) {
if (!(LIVE_HOOK_MS > 0) || row.status !== "active") return false;
const touched = Date.parse(row.updated_at);
if (!Number.isFinite(touched) || Date.now() - touched > LIVE_HOOK_MS) return false;
try {
let stmt = latestEventStmts.get(dbModule.db);
if (!stmt) {
stmt = dbModule.db.prepare(
"SELECT data FROM events WHERE session_id = ? ORDER BY id DESC LIMIT 1"
);
latestEventStmts.set(dbModule.db, stmt);
}
const ev = stmt.get(row.id);
return !!ev && !IMPORTED_DATA_RE.test(ev.data || "");
} catch {
return false; // when in doubt, sync as before
}
}
/**
* Incremental, change-driven sync of the default `~/.claude/projects` directory.
*
@@ -2024,6 +2065,25 @@ async function syncDefaultProjects(dbModule, options = {}) {
continue;
}
}
// A session hooks are feeding RIGHT NOW: its transcript keeps growing
// between hooks, so `mtime > updated_at` on nearly every sweep, and a
// full re-parse + snapshot copy here bought nothing the hooks do not
// already record. Leave it until it goes quiet (or ends): the sentinel
// keeps the watcher ignoring its appends, and never equals a real
// mtime, so the first sweep after the hooks stop catches it up.
if (existingRow && isHookLive(dbModule, existingRow)) {
mtimeCache.set(sourcePath, DEFERRED_MTIME);
continue;
}
// A session a hook created (no `imported` flag): importSession() skips
// it outright, so parsing the whole transcript + subagents was pure
// waste — only the durable snapshot copy still matters.
if (existingRow && !rowIsImported(existingRow)) {
mtimeCache.set(sourcePath, mtime);
if (require("../server/lib/run-as").sameOwner(sourcePath, existingRow.cwd))
await snapshotTranscript(sourcePath, sessionId);
continue;
}
const existed = !!existingRow;
const session = await parseSessionForImport(projPath, sourcePath);
// Record the mtime even when the file yields no session, so we don't