perf: cheaper ingest — no re-parse of live hooked transcripts, delta hook writes, lighter pollers

- Session sync leaves a transcript alone while hooks feed its session live
  (active, newest event a hook's, touched within DASHBOARD_SYNC_LIVE_HOOK_MS,
  default 5 min) and catches it up once quiet; a hook-created session (which
  importSession skips anyway) is only snapshotted, never parsed.
- Hook writer keeps a per-transcript mark (applied after commit): an
  unchanged TranscriptCache result skips token/compaction/API-error/turn
  writes, a grown one walks only the entries after the last one handled.
- Codex usage: one stamp query per pass instead of one row per journal;
  between full passes (first + every 5 min) only the newest day folders and
  journals written in the last hour are statted; fork parents resolved from
  the table or a full walk.
- Cursor/Antigravity polls back off to 60 s after idle passes
  (server/lib/backoff-poll.js); lane sync runs only while another dashboard
  shares the data dir (checked every 30 s).

server/lib/antigravity-usage.js

@@ -425,6 +425,9 @@ function writeSession(f, conv, now) {
 }
 
 let running = null;
+// Fresh sessions the last pass left for a later one to settle (the poll keeps
+// its base pace while there are any).
+let lastPending = 0;
 
 /**
  * One pass over every conversation file: a changed one is re-read, an
@@ -440,12 +443,16 @@ function ingestAntigravityUsage({
   running = (async () => {
     let read = 0;
     let written = 0;
+    let pending = 0; // still-fresh sessions a later pass may have to settle
     for (const f of conversationFiles()) {
       try {
         const seen = getSeen.get(f.path);
         if (seen && seen.mtime_ms === f.mtime) {
           const a = activity(f.id, f.mtime);
-          if (fresh(a, now)) continue;
+          if (fresh(a, now)) {
+            pending += 1;
+            continue;
+          }
           const s = stmts.getSession.get(f.id);
           if (s && s.tool === "antigravity" && s.status === "active") {
             const at = new Date(a.last).toISOString();
@@ -486,6 +493,7 @@ function ingestAntigravityUsage({
       }
       await new Promise((r) => setImmediate(r));
     }
+    lastPending = pending;
     return { files: read, sessions: written };
   })().finally(() => {
     running = null;
@@ -504,16 +512,16 @@ function startAntigravityUsagePoll(broadcast, broadcastLane) {
   if (!Number.isFinite(POLL_MS) || POLL_MS <= 0) return;
   // Files an older build could not read get another try.
   db.prepare("DELETE FROM antigravity_conversations WHERE session_id = ''").run();
-  const tick = () => {
-    ingestAntigravityUsage({ broadcast, broadcastLane })
-      .then((r) => {
+  // Slowed to once a minute after a pass that read nothing, wrote nothing and
+  // left no fresh session to settle; back to POLL_MS once one finds work.
+  require("./backoff-poll").startBackoffPoll(
+    () =>
+      ingestAntigravityUsage({ broadcast, broadcastLane }).then((r) => {
         if (r.sessions > 0 && pricingSync.syncEnabled()) void pricingSync.syncPricing(db);
-      })
-      .catch(() => {});
-  };
-  tick();
-  const timer = setInterval(tick, POLL_MS);
-  if (timer.unref) timer.unref();
+        return r.files + r.sessions + lastPending > 0;
+      }),
+    { baseMs: POLL_MS }
+  );
 }
 
 module.exports = {

server/lib/backoff-poll.js

@@ -0,0 +1,43 @@
+/**
+ * @file A background poll that slows down while it finds nothing to do: the
+ * next pass runs `baseMs` after a pass that did work (or failed), `idleMs`
+ * after one whose `tick` resolved `false` (nothing changed, nothing pending).
+ * One pass at a time — the next is scheduled only once the last settled — and
+ * every timer is `unref`'d. Used by the Cursor and Antigravity usage polls.
+ * @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
+ */
+
+/** Default slow interval: 60 s, never faster than the base one. */
+const idleDefault = (baseMs) => Math.max(baseMs, 60_000);
+
+/**
+ * Run `tick` now, then again and again. `tick` resolves `false` for an idle
+ * pass; anything else (or a throw) keeps the base pace.
+ * @returns {{stop: () => void}}
+ */
+function startBackoffPoll(tick, { baseMs, idleMs = idleDefault(baseMs) }) {
+  let timer = null;
+  let stopped = false;
+  const schedule = (ms) => {
+    if (stopped) return;
+    timer = setTimeout(run, ms);
+    if (timer.unref) timer.unref();
+  };
+  function run() {
+    Promise.resolve()
+      .then(tick)
+      .then(
+        (busy) => schedule(busy === false ? idleMs : baseMs),
+        () => schedule(baseMs)
+      );
+  }
+  run();
+  return {
+    stop() {
+      stopped = true;
+      clearTimeout(timer);
+    },
+  };
+}
+
+module.exports = { startBackoffPoll, idleDefault };

server/lib/codex-usage.js

@@ -235,7 +235,9 @@ function threadNames(home = getCodexHome()) {
   const names = new Map();
   let text = "";
   try {
-    text = require("./run-as").readFileSafe(path.join(home, "session_index.jsonl"), "utf8", { max: 16 << 20 });
+    text = require("./run-as").readFileSafe(path.join(home, "session_index.jsonl"), "utf8", {
+      max: 16 << 20,
+    });
   } catch {
     return names;
   }
@@ -251,6 +253,17 @@ function threadNames(home = getCodexHome()) {
 }
 
 const getRollout = db.prepare("SELECT * FROM codex_rollouts WHERE path = ?");
+// What a pass needs to skip an unchanged journal, for every journal at once:
+// one query per pass instead of one (with its whole parse state) per file.
+const rolloutStamps = db.prepare(
+  `SELECT path, size, mtime_ms, instr(state, '"more":true') > 0 AS more FROM codex_rollouts`
+);
+// Journals that may still be growing outside the newest day folders: written
+// within RECENT_ROW_MS, or not yet read to the end.
+const liveRolloutPaths = db.prepare(
+  `SELECT path FROM codex_rollouts WHERE mtime_ms > ? OR instr(state, '"more":true') > 0`
+);
+const threadPath = db.prepare("SELECT path FROM codex_rollouts WHERE thread_id = ? LIMIT 1");
 const upsertRollout = db.prepare(
   `INSERT INTO codex_rollouts (path, session_id, thread_id, size, mtime_ms, offset, state, updated_at)
    VALUES (?, ?, ?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
@@ -400,6 +413,59 @@ function writeSession(sessionId, names, now, origin = LOCAL) {
 
 const running = new Map(); // home ("" = CODEX_HOME) → the pass in flight
 
+// A RECENT pass (the poll's, between full ones) stats only the newest
+// RECENT_DAY_DIRS day folders plus the journals the table says may still grow
+// (RECENT_ROW_MS) — not every rollout Codex ever wrote. A journal in an older
+// folder that wakes up again (a resumed thread quiet for over RECENT_ROW_MS)
+// is picked up by the next full pass.
+const RECENT_DAY_DIRS = 2;
+const RECENT_ROW_MS = 60 * 60_000;
+
+function numericDirs(dir) {
+  try {
+    return fs
+      .readdirSync(dir, { withFileTypes: true })
+      .filter((e) => e.isDirectory() && /^\d+$/.test(e.name))
+      .map((e) => e.name)
+      .sort((a, b) => Number(b) - Number(a));
+  } catch {
+    return [];
+  }
+}
+
+function recentRolloutFiles(home, now) {
+  const root = path.join(home || getCodexHome(), "sessions");
+  const days = [];
+  outer: for (const y of numericDirs(root))
+    for (const m of numericDirs(path.join(root, y)))
+      for (const d of numericDirs(path.join(root, y, m))) {
+        days.push(path.join(root, y, m, d));
+        if (days.length >= RECENT_DAY_DIRS) break outer;
+      }
+  const paths = new Set();
+  for (const dir of days) {
+    try {
+      for (const e of fs.readdirSync(dir, { withFileTypes: true }))
+        if (e.isFile() && /^rollout-.*\.jsonl$/.test(e.name)) paths.add(path.join(dir, e.name));
+    } catch {
+      /* vanished mid-scan */
+    }
+  }
+  const prefix = root + path.sep;
+  for (const { path: p } of liveRolloutPaths.all(now - RECENT_ROW_MS))
+    if (p.startsWith(prefix)) paths.add(p);
+  const out = [];
+  for (const p of paths) {
+    try {
+      const stat = fs.statSync(p);
+      if (stat.isFile()) out.push({ path: p, mtime: stat.mtimeMs, size: stat.size });
+    } catch {
+      /* gone */
+    }
+  }
+  return out;
+}
+
 /**
  * One pass over every journal. Yields to the event loop between files; a
  * second call for the same home while one runs joins it. `home`, `source` and
@@ -413,21 +479,42 @@ function ingestCodexUsage({
   home,
   source = LOCAL.source,
   activeMs = LOCAL.activeMs,
+  recent = false,
 } = {}) {
   const key = home || "";
   if (running.has(key)) return running.get(key);
   const pass = (async () => {
-    const files = allRolloutFiles(home);
+    const files = recent ? recentRolloutFiles(home, now) : allRolloutFiles(home);
     const byThread = new Map();
     for (const f of files) {
       const m = FILE_NAME.exec(path.basename(f.path));
       if (m) byThread.set(m[1], f.path);
     }
+    // A recent pass lists only part of the tree: a fork's parent may live in
+    // an older folder, known to the table (or, failing that, to a full walk).
+    let walkedAll = !recent;
+    const parentPath = (id) => {
+      if (byThread.has(id)) return byThread.get(id);
+      if (walkedAll) return null;
+      const known = threadPath.get(id)?.path;
+      const root = path.join(home || getCodexHome(), "sessions") + path.sep;
+      if (known && known.startsWith(root) && fs.existsSync(known)) return known;
+      walkedAll = true;
+      for (const f of allRolloutFiles(home)) {
+        const m = FILE_NAME.exec(path.basename(f.path));
+        if (m && !byThread.has(m[1])) byThread.set(m[1], f.path);
+      }
+      return byThread.get(id) || null;
+    };
+    const stamps = new Map();
+    for (const r of rolloutStamps.all()) stamps.set(r.path, r);
     const touched = new Set();
     let read = 0;
     for (const f of files) {
       try {
         const mtime = Math.floor(f.mtime);
+        const seen = stamps.get(f.path);
+        if (seen && seen.size === f.size && seen.mtime_ms === mtime && !seen.more) continue;
         const row = getRollout.get(f.path);
         let offset = row ? row.offset : 0;
         let state = row ? JSON.parse(row.state) : {};
@@ -439,7 +526,8 @@ function ingestCodexUsage({
         let skip = null;
         const parent = offset === 0 ? firstLineMeta(f.path).forked_from_id : state.forkedFrom;
         if (parent && !state.pastFork) {
-          if (byThread.has(parent)) skip = await parentTotals(byThread.get(parent));
+          const pp = parentPath(parent);
+          if (pp) skip = await parentTotals(pp);
           else if (offset === 0)
             console.warn(`[codex-usage] fork parent ${parent} not found; ${f.path} may overcount`);
         }
@@ -508,10 +596,15 @@ function startCodexUsagePoll(broadcast) {
       .memberHomes()
       .map((h) => path.join(h, ".codex")),
   ];
+  // Every FULL_EVERY-th pass (and the first) walks every journal; the others
+  // only the recent ones (recentRolloutFiles).
+  const FULL_EVERY = Math.max(1, Math.round((5 * 60_000) / POLL_MS));
+  let passNo = 0;
   const ingestAll = async () => {
     const total = { sessions: 0 };
+    const recent = passNo++ % FULL_EVERY !== 0;
     for (const home of homes()) {
-      const r = await ingestCodexUsage({ broadcast, home });
+      const r = await ingestCodexUsage({ broadcast, home, recent });
       total.sessions += r?.sessions || 0;
     }
     return total;

server/lib/cursor-usage.js

@@ -182,14 +182,7 @@ function writeSession(f, now) {
   const endedAt = active ? null : lastWrite;
   const name = firstPrompt(f.path) || existing?.name || `Cursor ${f.id.slice(0, 8)}`;
   if (existing) {
-    updateSession.run(
-      name,
-      status,
-      endedAt,
-      lastWrite,
-      cwd,
-      f.id
-    );
+    updateSession.run(name, status, endedAt, lastWrite, cwd, f.id);
     return "session_updated";
   }
   insertSession.run(
@@ -206,6 +199,9 @@ function writeSession(f, now) {
 }
 
 let running = null;
+// Fresh sessions the last pass left for a later one to settle (the poll keeps
+// its base pace while there are any).
+let lastPending = 0;
 
 /**
  * One pass over every transcript: a changed one is re-read, an unchanged one
@@ -217,11 +213,15 @@ function ingestCursorUsage({ broadcast = () => {}, now = Date.now() } = {}) {
   running = (async () => {
     let read = 0;
     let written = 0;
+    let pending = 0; // still-fresh sessions a later pass may have to settle
     for (const f of transcriptFiles()) {
       try {
         const seen = getSeen.get(f.path);
         if (seen && seen.mtime_ms === f.mtime) {
-          if (fresh(f.mtime, now)) continue;
+          if (fresh(f.mtime, now)) {
+            pending += 1;
+            continue;
+          }
           const s = stmts.getSession.get(f.id);
           if (s && s.tool === "cursor" && s.status === "active") {
             settle.run(new Date(f.mtime).toISOString(), s.id);
@@ -242,6 +242,7 @@ function ingestCursorUsage({ broadcast = () => {}, now = Date.now() } = {}) {
       }
       await new Promise((r) => setImmediate(r));
     }
+    lastPending = pending;
     return { files: read, sessions: written };
   })().finally(() => {
     running = null;
@@ -249,18 +250,21 @@ function ingestCursorUsage({ broadcast = () => {}, now = Date.now() } = {}) {
   return running;
 }
 
-/** First pass at once (in the background), then every DASHBOARD_CURSOR_POLL_MS (default 15 s; 0 disables). */
+/**
+ * First pass at once (in the background), then every DASHBOARD_CURSOR_POLL_MS
+ * (default 15 s; 0 disables) — slowed to once a minute after a pass that read
+ * nothing, wrote nothing and left no fresh session to settle, and back to the
+ * base pace as soon as one finds work.
+ */
 function startCursorUsagePoll(broadcast) {
   const POLL_MS = process.env.DASHBOARD_CURSOR_POLL_MS
     ? Number(process.env.DASHBOARD_CURSOR_POLL_MS)
     : 15_000;
   if (!Number.isFinite(POLL_MS) || POLL_MS <= 0) return;
-  const tick = () => {
-    ingestCursorUsage({ broadcast }).catch(() => {});
-  };
-  tick();
-  const timer = setInterval(tick, POLL_MS);
-  if (timer.unref) timer.unref();
+  require("./backoff-poll").startBackoffPoll(
+    () => ingestCursorUsage({ broadcast }).then((r) => r.files + r.sessions + lastPending > 0),
+    { baseMs: POLL_MS }
+  );
 }
 
 module.exports = {