Merge branch 'perf/local-ingest' into perf/local-opt

This commit is contained in:
2026-10-16 10:20:59 +07:00
15 changed files with 734 additions and 51 deletions
+60
View File
@@ -1937,6 +1937,47 @@ async function ingestWorkflowsForBatch(dbModule, sessions) {
}
}
// Session sync: a session counts as fed LIVE by hooks while it is active, its
// newest event came from a hook (not this importer), and a hook touched it
// within this window. Such a transcript is left to the hooks until it goes
// quiet. DASHBOARD_SYNC_LIVE_HOOK_MS=0 turns the deferral off.
const LIVE_HOOK_MS = (() => {
const n = Number(process.env.DASHBOARD_SYNC_LIVE_HOOK_MS ?? 5 * 60_000);
return Number.isFinite(n) && n >= 0 ? n : 5 * 60_000;
})();
// mtimeCache value for a deferred file: present (so the fs watcher ignores its
// appends) yet never equal to a real mtime (so every sweep re-checks it).
const DEFERRED_MTIME = -1;
const IMPORTED_DATA_RE = /"imported"\s*:\s*true/;
const latestEventStmts = new WeakMap();
function rowIsImported(row) {
try {
return !!(row.metadata && JSON.parse(row.metadata).imported);
} catch {
return false;
}
}
function isHookLive(dbModule, row) {
if (!(LIVE_HOOK_MS > 0) || row.status !== "active") return false;
const touched = Date.parse(row.updated_at);
if (!Number.isFinite(touched) || Date.now() - touched > LIVE_HOOK_MS) return false;
try {
let stmt = latestEventStmts.get(dbModule.db);
if (!stmt) {
stmt = dbModule.db.prepare(
"SELECT data FROM events WHERE session_id = ? ORDER BY id DESC LIMIT 1"
);
latestEventStmts.set(dbModule.db, stmt);
}
const ev = stmt.get(row.id);
return !!ev && !IMPORTED_DATA_RE.test(ev.data || "");
} catch {
return false; // when in doubt, sync as before
}
}
/**
* Incremental, change-driven sync of the default `~/.claude/projects` directory.
*
@@ -2024,6 +2065,25 @@ async function syncDefaultProjects(dbModule, options = {}) {
continue;
}
}
// A session hooks are feeding RIGHT NOW: its transcript keeps growing
// between hooks, so `mtime > updated_at` on nearly every sweep, and a
// full re-parse + snapshot copy here bought nothing the hooks do not
// already record. Leave it until it goes quiet (or ends): the sentinel
// keeps the watcher ignoring its appends, and never equals a real
// mtime, so the first sweep after the hooks stop catches it up.
if (existingRow && isHookLive(dbModule, existingRow)) {
mtimeCache.set(sourcePath, DEFERRED_MTIME);
continue;
}
// A session a hook created (no `imported` flag): importSession() skips
// it outright, so parsing the whole transcript + subagents was pure
// waste — only the durable snapshot copy still matters.
if (existingRow && !rowIsImported(existingRow)) {
mtimeCache.set(sourcePath, mtime);
if (require("../server/lib/run-as").sameOwner(sourcePath, existingRow.cwd))
await snapshotTranscript(sourcePath, sessionId);
continue;
}
const existed = !!existingRow;
const session = await parseSessionForImport(projPath, sourcePath);
// Record the mtime even when the file yields no session, so we don't