perf: cheaper ingest — no re-parse of live hooked transcripts, delta hook writes, lighter pollers
- Session sync leaves a transcript alone while hooks feed its session live (active, newest event a hook's, touched within DASHBOARD_SYNC_LIVE_HOOK_MS, default 5 min) and catches it up once quiet; a hook-created session (which importSession skips anyway) is only snapshotted, never parsed. - Hook writer keeps a per-transcript mark (applied after commit): an unchanged TranscriptCache result skips token/compaction/API-error/turn writes, a grown one walks only the entries after the last one handled. - Codex usage: one stamp query per pass instead of one row per journal; between full passes (first + every 5 min) only the newest day folders and journals written in the last hour are statted; fork parents resolved from the table or a full walk. - Cursor/Antigravity polls back off to 60 s after idle passes (server/lib/backoff-poll.js); lane sync runs only while another dashboard shares the data dir (checked every 30 s).
This commit is contained in:
@@ -1937,6 +1937,47 @@ async function ingestWorkflowsForBatch(dbModule, sessions) {
|
||||
}
|
||||
}
|
||||
|
||||
// Session sync: a session counts as fed LIVE by hooks while it is active, its
|
||||
// newest event came from a hook (not this importer), and a hook touched it
|
||||
// within this window. Such a transcript is left to the hooks until it goes
|
||||
// quiet. DASHBOARD_SYNC_LIVE_HOOK_MS=0 turns the deferral off.
|
||||
const LIVE_HOOK_MS = (() => {
|
||||
const n = Number(process.env.DASHBOARD_SYNC_LIVE_HOOK_MS ?? 5 * 60_000);
|
||||
return Number.isFinite(n) && n >= 0 ? n : 5 * 60_000;
|
||||
})();
|
||||
// mtimeCache value for a deferred file: present (so the fs watcher ignores its
|
||||
// appends) yet never equal to a real mtime (so every sweep re-checks it).
|
||||
const DEFERRED_MTIME = -1;
|
||||
const IMPORTED_DATA_RE = /"imported"\s*:\s*true/;
|
||||
const latestEventStmts = new WeakMap();
|
||||
|
||||
function rowIsImported(row) {
|
||||
try {
|
||||
return !!(row.metadata && JSON.parse(row.metadata).imported);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function isHookLive(dbModule, row) {
|
||||
if (!(LIVE_HOOK_MS > 0) || row.status !== "active") return false;
|
||||
const touched = Date.parse(row.updated_at);
|
||||
if (!Number.isFinite(touched) || Date.now() - touched > LIVE_HOOK_MS) return false;
|
||||
try {
|
||||
let stmt = latestEventStmts.get(dbModule.db);
|
||||
if (!stmt) {
|
||||
stmt = dbModule.db.prepare(
|
||||
"SELECT data FROM events WHERE session_id = ? ORDER BY id DESC LIMIT 1"
|
||||
);
|
||||
latestEventStmts.set(dbModule.db, stmt);
|
||||
}
|
||||
const ev = stmt.get(row.id);
|
||||
return !!ev && !IMPORTED_DATA_RE.test(ev.data || "");
|
||||
} catch {
|
||||
return false; // when in doubt, sync as before
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Incremental, change-driven sync of the default `~/.claude/projects` directory.
|
||||
*
|
||||
@@ -2024,6 +2065,25 @@ async function syncDefaultProjects(dbModule, options = {}) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// A session hooks are feeding RIGHT NOW: its transcript keeps growing
|
||||
// between hooks, so `mtime > updated_at` on nearly every sweep, and a
|
||||
// full re-parse + snapshot copy here bought nothing the hooks do not
|
||||
// already record. Leave it until it goes quiet (or ends): the sentinel
|
||||
// keeps the watcher ignoring its appends, and never equals a real
|
||||
// mtime, so the first sweep after the hooks stop catches it up.
|
||||
if (existingRow && isHookLive(dbModule, existingRow)) {
|
||||
mtimeCache.set(sourcePath, DEFERRED_MTIME);
|
||||
continue;
|
||||
}
|
||||
// A session a hook created (no `imported` flag): importSession() skips
|
||||
// it outright, so parsing the whole transcript + subagents was pure
|
||||
// waste — only the durable snapshot copy still matters.
|
||||
if (existingRow && !rowIsImported(existingRow)) {
|
||||
mtimeCache.set(sourcePath, mtime);
|
||||
if (require("../server/lib/run-as").sameOwner(sourcePath, existingRow.cwd))
|
||||
await snapshotTranscript(sourcePath, sessionId);
|
||||
continue;
|
||||
}
|
||||
const existed = !!existingRow;
|
||||
const session = await parseSessionForImport(projPath, sourcePath);
|
||||
// Record the mtime even when the file yields no session, so we don't
|
||||
|
||||
Reference in New Issue
Block a user