perf: cheaper ingest — no re-parse of live hooked transcripts, delta hook writes, lighter pollers

- Session sync leaves a transcript alone while hooks feed its session live
  (active, newest event a hook's, touched within DASHBOARD_SYNC_LIVE_HOOK_MS,
  default 5 min) and catches it up once quiet; a hook-created session (which
  importSession skips anyway) is only snapshotted, never parsed.
- Hook writer keeps a per-transcript mark (applied after commit): an
  unchanged TranscriptCache result skips token/compaction/API-error/turn
  writes, a grown one walks only the entries after the last one handled.
- Codex usage: one stamp query per pass instead of one row per journal;
  between full passes (first + every 5 min) only the newest day folders and
  journals written in the last hour are statted; fork parents resolved from
  the table or a full walk.
- Cursor/Antigravity polls back off to 60 s after idle passes
  (server/lib/backoff-poll.js); lane sync runs only while another dashboard
  shares the data dir (checked every 30 s).

server/__tests__/backoff-poll.test.js

@@ -0,0 +1,42 @@
+/**
+ * @file Tests server/lib/backoff-poll.js — the poll that runs at its base pace
+ * while passes find work (or fail) and slows to its idle pace after a pass
+ * that resolved `false`, returning to the base pace once work shows up again.
+ * @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
+ */
+
+const { describe, it } = require("node:test");
+const assert = require("node:assert/strict");
+const { startBackoffPoll, idleDefault } = require("../lib/backoff-poll");
+
+const wait = (ms) => new Promise((r) => setTimeout(r, ms));
+
+describe("startBackoffPoll", () => {
+  it("slows down after idle passes and speeds up again on work", async () => {
+    const answers = [true, false, false, true, "x"];
+    const at = [];
+    const t0 = Date.now();
+    const poll = startBackoffPoll(
+      () => {
+        at.push(Date.now() - t0);
+        const a = answers.shift();
+        if (a === "x") throw new Error("boom");
+        return a;
+      },
+      { baseMs: 10, idleMs: 80 }
+    );
+    await wait(260);
+    poll.stop();
+    assert.ok(at.length >= 5, `ran ${at.length} times`);
+    const gaps = at.slice(1).map((t, i) => t - at[i]);
+    assert.ok(gaps[0] < 60, "work → base pace");
+    assert.ok(gaps[1] >= 75, "idle → idle pace");
+    assert.ok(gaps[2] >= 75, "still idle");
+    assert.ok(gaps[3] < 60, "work again → base pace");
+  });
+
+  it("idles at a minute by default, never faster than the base pace", () => {
+    assert.equal(idleDefault(15_000), 60_000);
+    assert.equal(idleDefault(120_000), 120_000);
+  });
+});

server/__tests__/codex-usage.test.js

@@ -536,3 +536,78 @@ describe("service tier", () => {
     ]);
   });
 });
+
+describe("ingestCodexUsage — recent pass", () => {
+  const NOW = Date.parse("2026-09-29T03:00:00Z");
+  function writeAt(day, id, lines, mtime) {
+    const dir = path.join(process.env.CODEX_HOME, "sessions", ...day.split("/"));
+    fs.mkdirSync(dir, { recursive: true });
+    const file = path.join(dir, `rollout-${day.replace(/\//g, "-")}T01-00-00-${id}.jsonl`);
+    fs.writeFileSync(file, jsonl(lines));
+    fs.utimesSync(file, mtime, mtime);
+    return file;
+  }
+
+  it("stats only the newest day folders and the journals still growing", async () => {
+    const quiet = writeAt(
+      "2026/09/01",
+      "old1",
+      [meta("old1"), ctx("gpt-a"), tc([10, 0, 1], [10, 0, 1])],
+      new Date(NOW - 20 * 86_400_000)
+    );
+    const live = writeAt(
+      "2026/09/02",
+      "old2",
+      [meta("old2"), ctx("gpt-a"), tc([10, 0, 1], [10, 0, 1])],
+      new Date(NOW - 60_000)
+    );
+    writeAt("2026/09/27", "a", [meta("a"), ctx("gpt-a")], new Date(NOW - 86_400_000));
+    writeAt("2026/09/28", "b", [meta("b"), ctx("gpt-a")], new Date(NOW - 86_400_000));
+    await ingestCodexUsage({ now: NOW }); // the full first pass
+
+    // Both old journals grow; only the one written within the hour is looked at.
+    fs.appendFileSync(quiet, jsonl([tc([30, 0, 3], [20, 0, 2])]));
+    fs.appendFileSync(live, jsonl([tc([30, 0, 3], [20, 0, 2])]));
+    fs.utimesSync(quiet, new Date(NOW - 1000), new Date(NOW - 1000));
+    fs.utimesSync(live, new Date(NOW - 1000), new Date(NOW - 1000));
+    // A new journal in today's folder is found too.
+    writeAt(
+      "2026/09/29",
+      "new1",
+      [meta("new1"), ctx("gpt-a"), tc([5, 0, 1], [5, 0, 1])],
+      new Date(NOW - 1000)
+    );
+    await ingestCodexUsage({ now: NOW, recent: true });
+    assert.deepEqual(usage("old2"), [{ model: "gpt-a", i: 30, r: 0, o: 3 }]);
+    assert.deepEqual(usage("old1"), [{ model: "gpt-a", i: 10, r: 0, o: 1 }], "left to a full pass");
+    assert.deepEqual(usage("new1"), [{ model: "gpt-a", i: 5, r: 0, o: 1 }]);
+
+    await ingestCodexUsage({ now: NOW }); // the next full pass catches it up
+    assert.deepEqual(usage("old1"), [{ model: "gpt-a", i: 30, r: 0, o: 3 }]);
+  });
+
+  it("finds a fork's parent in an older folder on a recent pass", async () => {
+    const parent = [
+      meta("p9"),
+      ctx("gpt-a"),
+      tc([100, 0, 10], [100, 0, 10]),
+      tc([300, 0, 30], [200, 0, 20]),
+    ];
+    writeAt("2026/08/01", "p9", parent, new Date(NOW - 50 * 86_400_000));
+    writeAt("2026/09/28", "x", [meta("x")], new Date(NOW - 86_400_000));
+    await ingestCodexUsage({ now: NOW });
+    writeAt(
+      "2026/09/29",
+      "f9",
+      [
+        meta("f9", { forked_from_id: "p9" }),
+        meta("p9"),
+        ...parent.slice(1),
+        tc([350, 0, 35], [50, 0, 5]),
+      ],
+      new Date(NOW - 1000)
+    );
+    await ingestCodexUsage({ now: NOW, recent: true });
+    assert.deepEqual(usage("f9"), [{ model: "gpt-a", i: 50, r: 0, o: 5 }]);
+  });
+});

server/__tests__/session-sync.test.js

@@ -150,3 +150,85 @@ describe("syncDefaultProjects", () => {
     );
   });
 });
+
+describe("syncDefaultProjects — sessions hooks are feeding", () => {
+  const SNAP_DIR = path.join(process.env.DASHBOARD_DATA_DIR, "transcripts");
+  const more = (sessionId, cwd, ts) => ({
+    type: "assistant",
+    cwd,
+    sessionId,
+    timestamp: ts,
+    message: {
+      model: "claude-opus-4-8",
+      content: [{ type: "text", text: "later" }],
+      usage: { input_tokens: 3, output_tokens: 2 },
+    },
+  });
+  const bumpMtime = (file) => {
+    const future = Date.now() / 1000 + 120;
+    fs.utimesSync(file, future, future);
+  };
+  const ageSession = (id) =>
+    dbModule.db
+      .prepare("UPDATE sessions SET updated_at = ? WHERE id = ?")
+      .run(new Date(Date.now() - 10 * 60_000).toISOString(), id);
+
+  it("defers a live hook-fed imported session, then catches it up once quiet", async () => {
+    const ID = "33333333-cccc-4ccc-8ccc-333333333333";
+    const cache = new Map();
+    const file = writeSession("-live", ID, fixtureLines(ID, "/live"));
+    await syncDefaultProjects(dbModule, { mtimeCache: cache });
+    assert.ok(dbModule.stmts.getSession.get(ID), "imported first");
+
+    // A hook arrives: its event is the newest, and it touches the session.
+    dbModule.stmts.insertEvent.run(ID, `${ID}-main`, "PreToolUse", "Bash", "hook", "{}");
+    dbModule.stmts.touchSession.run(ID);
+    writeSession(
+      "-live",
+      ID,
+      fixtureLines(ID, "/live", [more(ID, "/live", "2026-04-18T12:09:00.000Z")])
+    );
+    bumpMtime(file);
+
+    const live = await syncDefaultProjects(dbModule, { mtimeCache: cache });
+    assert.equal(
+      live.changed.find((c) => c.sessionId === ID),
+      undefined,
+      "deferred while live"
+    );
+    assert.equal(cache.get(file), -1, "kept in the cache so the watcher ignores its appends");
+
+    ageSession(ID); // the hooks went quiet
+    const quiet = await syncDefaultProjects(dbModule, { mtimeCache: cache });
+    assert.ok(
+      quiet.changed.find((c) => c.sessionId === ID && !c.isNew),
+      "caught up once quiet"
+    );
+    assert.notEqual(cache.get(file), -1);
+  });
+
+  it("never re-parses a hook-created session, only refreshes its snapshot once quiet", async () => {
+    const ID = "44444444-dddd-4ddd-8ddd-444444444444";
+    dbModule.stmts.insertSession.run(ID, "hooked", "active", "/hooked", null, null);
+    dbModule.stmts.insertEvent.run(ID, null, "SessionStart", null, "hook", "{}");
+    const file = writeSession("-hooked", ID, fixtureLines(ID, "/hooked"));
+    bumpMtime(file);
+    const snap = path.join(SNAP_DIR, `${ID}.jsonl`);
+    const cache = new Map();
+
+    await syncDefaultProjects(dbModule, { mtimeCache: cache });
+    assert.equal(fs.existsSync(snap), false, "no snapshot copy while hooks are live");
+
+    ageSession(ID);
+    const before = dbModule.db.prepare("SELECT COUNT(*) AS n FROM events WHERE session_id = ?");
+    const n0 = before.get(ID).n;
+    const r = await syncDefaultProjects(dbModule, { mtimeCache: cache });
+    assert.equal(
+      r.changed.find((c) => c.sessionId === ID),
+      undefined,
+      "nothing imported"
+    );
+    assert.equal(before.get(ID).n, n0, "no events written for a hook-created session");
+    assert.equal(fs.existsSync(snap), true, "snapshot taken once quiet");
+  });
+});

server/__tests__/transcript-marks.test.js

@@ -0,0 +1,167 @@
+/**
+ * @file Tests for the hook writer's per-transcript "already handled" marks
+ * (`transcriptMarks` in server/routes/hooks.js): a hook whose transcript is
+ * unchanged skips the token / compaction / API-error / turn-duration writes,
+ * a grown transcript is walked only from the last entry handled, and the rows
+ * left behind are exactly the ones a full re-walk would have written.
+ * @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
+ */
+
+const { describe, it, before, after } = require("node:test");
+const assert = require("node:assert/strict");
+const path = require("path");
+const fs = require("fs");
+const os = require("os");
+const http = require("http");
+
+const TMP = fs.mkdtempSync(path.join(os.tmpdir(), "agenthub-marks-"));
+process.env.DASHBOARD_DB_PATH = path.join(TMP, "dashboard.db");
+process.env.CLAUDE_HOME = path.join(TMP, "home");
+process.env.DASHBOARD_DATA_DIR = path.join(TMP, "data");
+
+const { createApp, startServer } = require("../index");
+const { db } = require("../db");
+const hooksRouter = require("../routes/hooks");
+
+let server;
+let BASE;
+
+function post(body) {
+  return new Promise((resolve, reject) => {
+    const url = new URL("/api/hooks/event", BASE);
+    const payload = JSON.stringify(body);
+    const r = http.request(
+      {
+        hostname: url.hostname,
+        port: url.port,
+        path: url.pathname,
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "Content-Length": Buffer.byteLength(payload),
+        },
+      },
+      (res) => {
+        res.resume();
+        res.on("end", () => resolve(res.statusCode));
+      }
+    );
+    r.on("error", reject);
+    r.write(payload);
+    r.end();
+  });
+}
+
+const line = (o) => JSON.stringify(o) + "\n";
+const assistant = (id, inTok) => ({
+  type: "assistant",
+  timestamp: "2026-09-18T01:00:00.000Z",
+  message: {
+    id,
+    model: "claude-sonnet-5",
+    content: [{ type: "text", text: "ok" }],
+    usage: { input_tokens: inTok, output_tokens: 1 },
+  },
+});
+const turn = (ms, ts) => ({
+  type: "system",
+  subtype: "turn_duration",
+  durationMs: ms,
+  timestamp: ts,
+});
+const apiError = (text, ts) => ({
+  type: "assistant",
+  isApiErrorMessage: true,
+  error: "rate_limit",
+  timestamp: ts,
+  message: { content: [{ type: "text", text }] },
+});
+const compact = (uuid, ts) => ({ type: "user", isCompactSummary: true, uuid, timestamp: ts });
+
+function rows(sid) {
+  const ev = db
+    .prepare(
+      "SELECT event_type, summary, created_at FROM events WHERE session_id = ? AND event_type IN ('TurnDuration','APIError','Compaction') ORDER BY event_type, created_at, summary"
+    )
+    .all(sid)
+    .map((r) =>
+      r.event_type === "APIError"
+        ? `${r.event_type}|${r.summary}`
+        : `${r.event_type}|${r.created_at}|${r.summary}`
+    );
+  const tokens = db
+    .prepare("SELECT model, input_tokens, baseline_input FROM token_usage WHERE session_id = ?")
+    .all(sid);
+  return { ev, tokens };
+}
+
+before(async () => {
+  server = await startServer(createApp(), 0);
+  BASE = `http://127.0.0.1:${server.address().port}`;
+});
+
+after(() => {
+  server?.close();
+  fs.rmSync(TMP, { recursive: true, force: true });
+});
+
+describe("transcript marks", () => {
+  it("skips an unchanged transcript and walks only the new tail of a grown one", async () => {
+    const sid = "abababab-1111-4111-8111-abababababab";
+    const cwd = path.join(TMP, "work");
+    fs.mkdirSync(cwd, { recursive: true });
+    const tpath = path.join(TMP, `${sid}.jsonl`);
+    fs.writeFileSync(
+      tpath,
+      line(assistant("m1", 5)) +
+        line(turn(1000, "2026-09-18T01:00:01.000Z")) +
+        line(apiError("slow down", "2026-09-18T01:00:02.000Z")) +
+        line(compact("c-1", "2026-09-18T01:00:03.000Z")) +
+        line(assistant("m2", 7)) +
+        line(turn(2000, "2026-09-18T01:00:04.000Z"))
+    );
+    const data = { session_id: sid, cwd, transcript_path: tpath };
+
+    assert.equal(await post({ hook_type: "PostToolUse", data }), 200);
+    const first = rows(sid);
+    assert.equal(first.ev.filter((e) => e.startsWith("TurnDuration")).length, 2);
+    assert.equal(first.ev.filter((e) => e.startsWith("APIError")).length, 1);
+    assert.equal(first.ev.filter((e) => e.startsWith("Compaction")).length, 1);
+    assert.ok(hooksRouter.transcriptMarks.size >= 1, "a mark is kept after commit");
+
+    // Proof of the skip: a row removed behind the writer's back is NOT put
+    // back by a hook whose transcript did not change (before, every hook
+    // re-walked the whole history and re-inserted it).
+    db.prepare(
+      "DELETE FROM events WHERE session_id = ? AND event_type = 'TurnDuration' AND created_at = '2026-09-18T01:00:01.000Z'"
+    ).run(sid);
+    assert.equal(await post({ hook_type: "PostToolUse", data: { ...data, tool_name: "x" } }), 200);
+    assert.equal(rows(sid).ev.filter((e) => e.startsWith("TurnDuration")).length, 1);
+    db.prepare("DELETE FROM events WHERE session_id = ? AND event_type = 'TurnDuration'").run(sid);
+    hooksRouter.transcriptMarks.clear();
+    assert.equal(await post({ hook_type: "PostToolUse", data: { ...data, tool_name: "y" } }), 200);
+    assert.deepEqual(rows(sid), first, "a cleared mark re-walks the whole history, as before");
+
+    // Grow it: one turn, one new error, a repeat of the old error, a compaction.
+    fs.appendFileSync(
+      tpath,
+      line(assistant("m3", 11)) +
+        line(turn(3000, "2026-09-18T01:00:05.000Z")) +
+        line(apiError("slow down", "2026-09-18T01:00:06.000Z")) +
+        line(apiError("overloaded", "2026-09-18T01:00:07.000Z")) +
+        line(compact("c-2", "2026-09-18T01:00:08.000Z"))
+    );
+    assert.equal(await post({ hook_type: "Stop", data }), 200);
+    const grown = rows(sid);
+
+    // The same transcript walked from scratch must produce the same rows.
+    hooksRouter.transcriptMarks.clear();
+    assert.equal(await post({ hook_type: "Stop", data: { ...data, stop_hook_active: true } }), 200);
+    assert.deepEqual(rows(sid), grown, "the delta wrote exactly what a full walk writes");
+    assert.equal(grown.ev.filter((e) => e.startsWith("TurnDuration")).length, 3);
+    assert.equal(grown.ev.filter((e) => e.startsWith("APIError")).length, 2, "repeat deduped");
+    assert.equal(grown.ev.filter((e) => e.startsWith("Compaction")).length, 2);
+    assert.equal(grown.tokens.length, 1);
+    assert.equal(grown.tokens[0].input_tokens + grown.tokens[0].baseline_input, 23);
+  });
+});