perf: cheaper ingest — no re-parse of live hooked transcripts, delta hook writes, lighter pollers

- Session sync leaves a transcript alone while hooks feed its session live (active, newest event a hook's, touched within DASHBOARD_SYNC_LIVE_HOOK_MS, default 5 min) and catches it up once quiet; a hook-created session (which importSession skips anyway) is only snapshotted, never parsed. - Hook writer keeps a per-transcript mark (applied after commit): an unchanged TranscriptCache result skips token/compaction/API-error/turn writes, a grown one walks only the entries after the last one handled. - Codex usage: one stamp query per pass instead of one row per journal; between full passes (first + every 5 min) only the newest day folders and journals written in the last hour are statted; fork parents resolved from the table or a full walk. - Cursor/Antigravity polls back off to 60 s after idle passes (server/lib/backoff-poll.js); lane sync runs only while another dashboard shares the data dir (checked every 30 s).
2026-10-16 10:20:09 +07:00
parent ecbbb4dd8f
commit 5cb70d702b
15 changed files with 734 additions and 51 deletions
@@ -0,0 +1,167 @@
+/**
+ * @file Tests for the hook writer's per-transcript "already handled" marks
+ * (`transcriptMarks` in server/routes/hooks.js): a hook whose transcript is
+ * unchanged skips the token / compaction / API-error / turn-duration writes,
+ * a grown transcript is walked only from the last entry handled, and the rows
+ * left behind are exactly the ones a full re-walk would have written.
+ * @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
+ */
+
+const { describe, it, before, after } = require("node:test");
+const assert = require("node:assert/strict");
+const path = require("path");
+const fs = require("fs");
+const os = require("os");
+const http = require("http");
+
+const TMP = fs.mkdtempSync(path.join(os.tmpdir(), "agenthub-marks-"));
+process.env.DASHBOARD_DB_PATH = path.join(TMP, "dashboard.db");
+process.env.CLAUDE_HOME = path.join(TMP, "home");
+process.env.DASHBOARD_DATA_DIR = path.join(TMP, "data");
+
+const { createApp, startServer } = require("../index");
+const { db } = require("../db");
+const hooksRouter = require("../routes/hooks");
+
+let server;
+let BASE;
+
+function post(body) {
+  return new Promise((resolve, reject) => {
+    const url = new URL("/api/hooks/event", BASE);
+    const payload = JSON.stringify(body);
+    const r = http.request(
+      {
+        hostname: url.hostname,
+        port: url.port,
+        path: url.pathname,
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "Content-Length": Buffer.byteLength(payload),
+        },
+      },
+      (res) => {
+        res.resume();
+        res.on("end", () => resolve(res.statusCode));
+      }
+    );
+    r.on("error", reject);
+    r.write(payload);
+    r.end();
+  });
+}
+
+const line = (o) => JSON.stringify(o) + "\n";
+const assistant = (id, inTok) => ({
+  type: "assistant",
+  timestamp: "2026-09-18T01:00:00.000Z",
+  message: {
+    id,
+    model: "claude-sonnet-5",
+    content: [{ type: "text", text: "ok" }],
+    usage: { input_tokens: inTok, output_tokens: 1 },
+  },
+});
+const turn = (ms, ts) => ({
+  type: "system",
+  subtype: "turn_duration",
+  durationMs: ms,
+  timestamp: ts,
+});
+const apiError = (text, ts) => ({
+  type: "assistant",
+  isApiErrorMessage: true,
+  error: "rate_limit",
+  timestamp: ts,
+  message: { content: [{ type: "text", text }] },
+});
+const compact = (uuid, ts) => ({ type: "user", isCompactSummary: true, uuid, timestamp: ts });
+
+function rows(sid) {
+  const ev = db
+    .prepare(
+      "SELECT event_type, summary, created_at FROM events WHERE session_id = ? AND event_type IN ('TurnDuration','APIError','Compaction') ORDER BY event_type, created_at, summary"
+    )
+    .all(sid)
+    .map((r) =>
+      r.event_type === "APIError"
+        ? `${r.event_type}|${r.summary}`
+        : `${r.event_type}|${r.created_at}|${r.summary}`
+    );
+  const tokens = db
+    .prepare("SELECT model, input_tokens, baseline_input FROM token_usage WHERE session_id = ?")
+    .all(sid);
+  return { ev, tokens };
+}
+
+before(async () => {
+  server = await startServer(createApp(), 0);
+  BASE = `http://127.0.0.1:${server.address().port}`;
+});
+
+after(() => {
+  server?.close();
+  fs.rmSync(TMP, { recursive: true, force: true });
+});
+
+describe("transcript marks", () => {
+  it("skips an unchanged transcript and walks only the new tail of a grown one", async () => {
+    const sid = "abababab-1111-4111-8111-abababababab";
+    const cwd = path.join(TMP, "work");
+    fs.mkdirSync(cwd, { recursive: true });
+    const tpath = path.join(TMP, `${sid}.jsonl`);
+    fs.writeFileSync(
+      tpath,
+      line(assistant("m1", 5)) +
+        line(turn(1000, "2026-09-18T01:00:01.000Z")) +
+        line(apiError("slow down", "2026-09-18T01:00:02.000Z")) +
+        line(compact("c-1", "2026-09-18T01:00:03.000Z")) +
+        line(assistant("m2", 7)) +
+        line(turn(2000, "2026-09-18T01:00:04.000Z"))
+    );
+    const data = { session_id: sid, cwd, transcript_path: tpath };
+
+    assert.equal(await post({ hook_type: "PostToolUse", data }), 200);
+    const first = rows(sid);
+    assert.equal(first.ev.filter((e) => e.startsWith("TurnDuration")).length, 2);
+    assert.equal(first.ev.filter((e) => e.startsWith("APIError")).length, 1);
+    assert.equal(first.ev.filter((e) => e.startsWith("Compaction")).length, 1);
+    assert.ok(hooksRouter.transcriptMarks.size >= 1, "a mark is kept after commit");
+
+    // Proof of the skip: a row removed behind the writer's back is NOT put
+    // back by a hook whose transcript did not change (before, every hook
+    // re-walked the whole history and re-inserted it).
+    db.prepare(
+      "DELETE FROM events WHERE session_id = ? AND event_type = 'TurnDuration' AND created_at = '2026-09-18T01:00:01.000Z'"
+    ).run(sid);
+    assert.equal(await post({ hook_type: "PostToolUse", data: { ...data, tool_name: "x" } }), 200);
+    assert.equal(rows(sid).ev.filter((e) => e.startsWith("TurnDuration")).length, 1);
+    db.prepare("DELETE FROM events WHERE session_id = ? AND event_type = 'TurnDuration'").run(sid);
+    hooksRouter.transcriptMarks.clear();
+    assert.equal(await post({ hook_type: "PostToolUse", data: { ...data, tool_name: "y" } }), 200);
+    assert.deepEqual(rows(sid), first, "a cleared mark re-walks the whole history, as before");
+
+    // Grow it: one turn, one new error, a repeat of the old error, a compaction.
+    fs.appendFileSync(
+      tpath,
+      line(assistant("m3", 11)) +
+        line(turn(3000, "2026-09-18T01:00:05.000Z")) +
+        line(apiError("slow down", "2026-09-18T01:00:06.000Z")) +
+        line(apiError("overloaded", "2026-09-18T01:00:07.000Z")) +
+        line(compact("c-2", "2026-09-18T01:00:08.000Z"))
+    );
+    assert.equal(await post({ hook_type: "Stop", data }), 200);
+    const grown = rows(sid);
+
+    // The same transcript walked from scratch must produce the same rows.
+    hooksRouter.transcriptMarks.clear();
+    assert.equal(await post({ hook_type: "Stop", data: { ...data, stop_hook_active: true } }), 200);
+    assert.deepEqual(rows(sid), grown, "the delta wrote exactly what a full walk writes");
+    assert.equal(grown.ev.filter((e) => e.startsWith("TurnDuration")).length, 3);
+    assert.equal(grown.ev.filter((e) => e.startsWith("APIError")).length, 2, "repeat deduped");
+    assert.equal(grown.ev.filter((e) => e.startsWith("Compaction")).length, 2);
+    assert.equal(grown.tokens.length, 1);
+    assert.equal(grown.tokens[0].input_tokens + grown.tokens[0].baseline_input, 23);
+  });
+});