perf: cheaper ingest — no re-parse of live hooked transcripts, delta hook writes, lighter pollers

- Session sync leaves a transcript alone while hooks feed its session live
  (active, newest event a hook's, touched within DASHBOARD_SYNC_LIVE_HOOK_MS,
  default 5 min) and catches it up once quiet; a hook-created session (which
  importSession skips anyway) is only snapshotted, never parsed.
- Hook writer keeps a per-transcript mark (applied after commit): an
  unchanged TranscriptCache result skips token/compaction/API-error/turn
  writes, a grown one walks only the entries after the last one handled.
- Codex usage: one stamp query per pass instead of one row per journal;
  between full passes (first + every 5 min) only the newest day folders and
  journals written in the last hour are statted; fork parents resolved from
  the table or a full walk.
- Cursor/Antigravity polls back off to 60 s after idle passes
  (server/lib/backoff-poll.js); lane sync runs only while another dashboard
  shares the data dir (checked every 30 s).

server/lib/codex-usage.js

@@ -235,7 +235,9 @@ function threadNames(home = getCodexHome()) {
   const names = new Map();
   let text = "";
   try {
-    text = require("./run-as").readFileSafe(path.join(home, "session_index.jsonl"), "utf8", { max: 16 << 20 });
+    text = require("./run-as").readFileSafe(path.join(home, "session_index.jsonl"), "utf8", {
+      max: 16 << 20,
+    });
   } catch {
     return names;
   }
@@ -251,6 +253,17 @@ function threadNames(home = getCodexHome()) {
 }
 
 const getRollout = db.prepare("SELECT * FROM codex_rollouts WHERE path = ?");
+// What a pass needs to skip an unchanged journal, for every journal at once:
+// one query per pass instead of one (with its whole parse state) per file.
+const rolloutStamps = db.prepare(
+  `SELECT path, size, mtime_ms, instr(state, '"more":true') > 0 AS more FROM codex_rollouts`
+);
+// Journals that may still be growing outside the newest day folders: written
+// within RECENT_ROW_MS, or not yet read to the end.
+const liveRolloutPaths = db.prepare(
+  `SELECT path FROM codex_rollouts WHERE mtime_ms > ? OR instr(state, '"more":true') > 0`
+);
+const threadPath = db.prepare("SELECT path FROM codex_rollouts WHERE thread_id = ? LIMIT 1");
 const upsertRollout = db.prepare(
   `INSERT INTO codex_rollouts (path, session_id, thread_id, size, mtime_ms, offset, state, updated_at)
    VALUES (?, ?, ?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
@@ -400,6 +413,59 @@ function writeSession(sessionId, names, now, origin = LOCAL) {
 
 const running = new Map(); // home ("" = CODEX_HOME) → the pass in flight
 
+// A RECENT pass (the poll's, between full ones) stats only the newest
+// RECENT_DAY_DIRS day folders plus the journals the table says may still grow
+// (RECENT_ROW_MS) — not every rollout Codex ever wrote. A journal in an older
+// folder that wakes up again (a resumed thread quiet for over RECENT_ROW_MS)
+// is picked up by the next full pass.
+const RECENT_DAY_DIRS = 2;
+const RECENT_ROW_MS = 60 * 60_000;
+
+function numericDirs(dir) {
+  try {
+    return fs
+      .readdirSync(dir, { withFileTypes: true })
+      .filter((e) => e.isDirectory() && /^\d+$/.test(e.name))
+      .map((e) => e.name)
+      .sort((a, b) => Number(b) - Number(a));
+  } catch {
+    return [];
+  }
+}
+
+function recentRolloutFiles(home, now) {
+  const root = path.join(home || getCodexHome(), "sessions");
+  const days = [];
+  outer: for (const y of numericDirs(root))
+    for (const m of numericDirs(path.join(root, y)))
+      for (const d of numericDirs(path.join(root, y, m))) {
+        days.push(path.join(root, y, m, d));
+        if (days.length >= RECENT_DAY_DIRS) break outer;
+      }
+  const paths = new Set();
+  for (const dir of days) {
+    try {
+      for (const e of fs.readdirSync(dir, { withFileTypes: true }))
+        if (e.isFile() && /^rollout-.*\.jsonl$/.test(e.name)) paths.add(path.join(dir, e.name));
+    } catch {
+      /* vanished mid-scan */
+    }
+  }
+  const prefix = root + path.sep;
+  for (const { path: p } of liveRolloutPaths.all(now - RECENT_ROW_MS))
+    if (p.startsWith(prefix)) paths.add(p);
+  const out = [];
+  for (const p of paths) {
+    try {
+      const stat = fs.statSync(p);
+      if (stat.isFile()) out.push({ path: p, mtime: stat.mtimeMs, size: stat.size });
+    } catch {
+      /* gone */
+    }
+  }
+  return out;
+}
+
 /**
  * One pass over every journal. Yields to the event loop between files; a
  * second call for the same home while one runs joins it. `home`, `source` and
@@ -413,21 +479,42 @@ function ingestCodexUsage({
   home,
   source = LOCAL.source,
   activeMs = LOCAL.activeMs,
+  recent = false,
 } = {}) {
   const key = home || "";
   if (running.has(key)) return running.get(key);
   const pass = (async () => {
-    const files = allRolloutFiles(home);
+    const files = recent ? recentRolloutFiles(home, now) : allRolloutFiles(home);
     const byThread = new Map();
     for (const f of files) {
       const m = FILE_NAME.exec(path.basename(f.path));
       if (m) byThread.set(m[1], f.path);
     }
+    // A recent pass lists only part of the tree: a fork's parent may live in
+    // an older folder, known to the table (or, failing that, to a full walk).
+    let walkedAll = !recent;
+    const parentPath = (id) => {
+      if (byThread.has(id)) return byThread.get(id);
+      if (walkedAll) return null;
+      const known = threadPath.get(id)?.path;
+      const root = path.join(home || getCodexHome(), "sessions") + path.sep;
+      if (known && known.startsWith(root) && fs.existsSync(known)) return known;
+      walkedAll = true;
+      for (const f of allRolloutFiles(home)) {
+        const m = FILE_NAME.exec(path.basename(f.path));
+        if (m && !byThread.has(m[1])) byThread.set(m[1], f.path);
+      }
+      return byThread.get(id) || null;
+    };
+    const stamps = new Map();
+    for (const r of rolloutStamps.all()) stamps.set(r.path, r);
     const touched = new Set();
     let read = 0;
     for (const f of files) {
       try {
         const mtime = Math.floor(f.mtime);
+        const seen = stamps.get(f.path);
+        if (seen && seen.size === f.size && seen.mtime_ms === mtime && !seen.more) continue;
         const row = getRollout.get(f.path);
         let offset = row ? row.offset : 0;
         let state = row ? JSON.parse(row.state) : {};
@@ -439,7 +526,8 @@ function ingestCodexUsage({
         let skip = null;
         const parent = offset === 0 ? firstLineMeta(f.path).forked_from_id : state.forkedFrom;
         if (parent && !state.pastFork) {
-          if (byThread.has(parent)) skip = await parentTotals(byThread.get(parent));
+          const pp = parentPath(parent);
+          if (pp) skip = await parentTotals(pp);
           else if (offset === 0)
             console.warn(`[codex-usage] fork parent ${parent} not found; ${f.path} may overcount`);
         }
@@ -508,10 +596,15 @@ function startCodexUsagePoll(broadcast) {
       .memberHomes()
       .map((h) => path.join(h, ".codex")),
   ];
+  // Every FULL_EVERY-th pass (and the first) walks every journal; the others
+  // only the recent ones (recentRolloutFiles).
+  const FULL_EVERY = Math.max(1, Math.round((5 * 60_000) / POLL_MS));
+  let passNo = 0;
   const ingestAll = async () => {
     const total = { sessions: 0 };
+    const recent = passNo++ % FULL_EVERY !== 0;
     for (const home of homes()) {
-      const r = await ingestCodexUsage({ broadcast, home });
+      const r = await ingestCodexUsage({ broadcast, home, recent });
       total.sessions += r?.sessions || 0;
     }
     return total;