perf: v2.5.10 — ready for 10+ concurrent users, leaner locally

Measured on a copy of a real 1.2 GB / 253K-event database, 10 simulated browsers + 10 hooks/s for 90 s: event-loop delay p99 1239 ms → 32 ms (max 2.0 s → 94 ms), 266 failed requests → 0, 74 → 111 requests/s served, server RSS 633 → 346 MB. Entry bundle 975 KB raw → 394 KB (106 KB brotli). Server - /api/analytics: the events-table scans run in the read worker (scoped-stats eventAggregates, read-pool job analyticsEvents); runRead takes maxAgeMs — analytics 10 s, /api/workflows 30 s — keyed by the args - /api/workflows/skill-usage: +event_type and a tool_name range instead of LIKE, so it uses idx_events_tool (784 ms → a few ms) - idx_events_session_type_tool (session_id, event_type, tool_name, created_at) replaces idx_events_session_type: covering reads for session stats, the hook dedupes and the import resume point (1.2 s → 20 ms); 64 MB cache_size and 256 MB mmap_size on both connections - WAL: the server sets wal_autocheckpoint = 0 and checkpoints PASSIVE every DASHBOARD_WAL_CHECKPOINT_MS (5 s) from the read worker's own writable connection; a failed checkpoint restores the automatic one (hook COMMIT p95 109 ms → 15 ms at 100 hooks/s) - tmux: one list-sessions answers a whole GET /api/lanes and the heal beat (liveSessions), shared for 1 s and cleared by this process's new/kill — GET /api/run's sync spawns had blocked 60 s of a 90 s run; the heal beat skips tmux when nothing runs; attach options chained into 2 spawns - hooks: TurnDuration/APIError deduped against one read per hook instead of a query per transcript entry (23 ms/hook on a 763-turn session); one lane scan per hook; hourly upserts skip rows that did not grow; frames are sent after the transaction commits (and after the pane binding); new_event carries its row id; a split pane's (slot 1–9) hooks no longer rebind the lane's session_id - watchdog liveness probe runs ps asynchronously; transcript newline scan via indexOf; startup subagent backfill fingerprints the files it already checked - pty attach: backpressure (pause past 1 MB unsent, resume under 256 KB) - static: hashed assets served from their pre-compressed .br/.gz sibling Client - hidden tabs neither refetch on WS events nor poll (useLiveRefresh, panes, lane cards, git header, changes panel, settings); no per-tab hourly /updates/check (the server already checks) - refetch storms throttled (workspace tree/session list 3 s, /api/run only on a run-key change, notification pane lookup only when shown); WS reconnect jitter; terminal retry backoff 1.5 → 30 s, stops on 1008 - heavier throttles for aggregates (15 s) and detail views; Monitor keepIfSame; GET in-flight dedupe in request() - lazy Dashboard/History, xterm (with the first terminal), lane modals, Sagi (only while enabled); only the active locale loads; Workspace chunk prefetched on a /workspace deep link - Sagi: no work while disabled, pupil via CSS variables, memo avatar - build: .br/.gz siblings, sourcemap hidden; service worker no longer caches /assets/ (cache v3 drops the old one) Tests: scripts/test-server.js drops the GIT_DIR-family variables a hook exports, so a suite run from a linked worktree's pre-commit can no longer commit into the real repository or set core.bare. Docs: ARCHITECTURE, DATABASE, API, DEPLOYMENT, client README, docs/plans/perf/README.md (audit, checklist, results).
2026-10-07 13:24:15 +07:00
parent 8fdda66893
commit ca821008e2
75 changed files with 1711 additions and 699 deletions
@@ -1813,11 +1813,13 @@ describe("Database Integrity", () => {

    assert.ok(indexes.includes("idx_agents_session"));
    assert.ok(indexes.includes("idx_agents_status"));
-    // (session_id, id) is NOT a prefix of idx_events_session_type: the hook
+    // (session_id, id) is NOT a prefix of idx_events_session_type_tool: the hook
    // duplicate guard reads a session's latest event by id through it.
    assert.ok(indexes.includes("idx_events_session_created"));
    assert.ok(indexes.includes("idx_events_session"));
    assert.ok(indexes.includes("idx_events_type"));
+    assert.ok(indexes.includes("idx_events_session_type_tool"));
+    assert.ok(!indexes.includes("idx_events_session_type"));
    assert.ok(indexes.includes("idx_events_created"));
    assert.ok(indexes.includes("idx_sessions_status"));
    assert.ok(indexes.includes("idx_sessions_started"));
@@ -2,7 +2,8 @@
 * @file Tests the DASHBOARD_CLIENT_DIST override in server/index.js. A plugin
 * install runs the server from a read-only plugin cache directory, so the
 * client bundle is served from the writable runtime dir instead of the
- * checkout's client/dist. Also asserts the API still answers when the
+ * checkout's client/dist, with a hashed asset's pre-compressed `.br`/`.gz`
+ * sibling sent to a browser that accepts it. Also asserts the API still answers when the
 * configured bundle directory does not exist yet (the normal state before
 * /agenthub-open builds it).
 * @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
@@ -19,6 +20,16 @@ const TMP = fs.mkdtempSync(path.join(os.tmpdir(), "agenthub-clientdist-"));
 const DIST = path.join(TMP, "client-dist");
 fs.mkdirSync(DIST);
 fs.writeFileSync(path.join(DIST, "index.html"), "<!doctype html><title>from-runtime</title>");
+// A hashed bundle with the pre-compressed siblings the client build writes.
+fs.mkdirSync(path.join(DIST, "assets"));
+const BUNDLE = "console.log('plain bundle');".repeat(50);
+fs.writeFileSync(path.join(DIST, "assets", "app-abc123.js"), BUNDLE);
+fs.writeFileSync(
+  path.join(DIST, "assets", "app-abc123.js.br"),
+  require("zlib").brotliCompressSync(BUNDLE)
+);
+fs.writeFileSync(path.join(DIST, "assets", "app-abc123.js.gz"), require("zlib").gzipSync(BUNDLE));
+fs.writeFileSync(path.join(DIST, "assets", "only-plain.css"), "body{}");

 // Must be set BEFORE requiring the server: the data dir, discovery file and
 // the static mount are all resolved at startup. NODE_ENV=production is the
@@ -38,12 +49,15 @@ const { db } = require("../db");
 let server;
 let BASE;

-function get(urlPath) {
+function get(urlPath, headers = {}) {
  return new Promise((resolve, reject) => {
-    const req = http.get(new URL(urlPath, BASE), (res) => {
-      let body = "";
-      res.on("data", (c) => (body += c));
-      res.on("end", () => resolve({ status: res.statusCode, body }));
+    const req = http.get(new URL(urlPath, BASE), { headers }, (res) => {
+      const chunks = [];
+      res.on("data", (c) => chunks.push(c));
+      res.on("end", () => {
+        const raw = Buffer.concat(chunks);
+        resolve({ status: res.statusCode, body: raw.toString(), raw, headers: res.headers });
+      });
    });
    req.on("error", reject);
  });
@@ -68,6 +82,33 @@ describe("DASHBOARD_CLIENT_DIST override", () => {
    assert.match(res.body, /from-runtime/);
  });

+  it("serves a hashed asset's pre-compressed sibling to a browser that takes it", async () => {
+    const zlib = require("zlib");
+    const br = await get("/assets/app-abc123.js", { "accept-encoding": "gzip, deflate, br" });
+    assert.equal(br.status, 200);
+    assert.equal(br.headers["content-encoding"], "br");
+    assert.match(br.headers["content-type"], /javascript/);
+    assert.match(br.headers.vary, /Accept-Encoding/i);
+    assert.match(br.headers["cache-control"], /immutable/);
+    assert.equal(zlib.brotliDecompressSync(br.raw).toString(), BUNDLE);
+
+    const gz = await get("/assets/app-abc123.js", { "accept-encoding": "gzip" });
+    assert.equal(gz.headers["content-encoding"], "gzip");
+    assert.equal(zlib.gunzipSync(gz.raw).toString(), BUNDLE);
+
+    const plain = await get("/assets/app-abc123.js", { "accept-encoding": "identity" });
+    assert.equal(plain.headers["content-encoding"], undefined);
+    assert.equal(plain.body, BUNDLE);
+
+    // No sibling on disk: the plain file, as before.
+    const css = await get("/assets/only-plain.css", { "accept-encoding": "br" });
+    assert.equal(css.headers["content-encoding"], undefined);
+    assert.equal(css.body, "body{}");
+    // A name that is not a flat file name never reaches the disk lookup.
+    const walk = await get("/assets/..%2findex.html", { "accept-encoding": "br" });
+    assert.equal(walk.headers["content-encoding"], undefined);
+  });
+
  it("keeps the API working alongside the override", async () => {
    const res = await get("/api/health");
    assert.equal(res.status, 200);
@@ -541,6 +541,30 @@ describe("hook → lane binding", () => {
    assert.equal((await request("GET", "/api/lanes")).body.lanes.length, before);
  });

+  it("keeps the lane's session when a split pane (slot 1–9) of the workspace sends hooks", async () => {
+    const created = await request("POST", "/api/lanes", { cwd: "/tmp/lane-split-pane" });
+    const id = created.body.lane.id;
+    const hook = (session_id, run_id) =>
+      request("POST", "/api/hooks/event", {
+        hook_type: "PostToolUse",
+        run_id,
+        data: {
+          session_id,
+          cwd: "/tmp/lane-split-pane",
+          tool_name: "Read",
+          tool_use_id: session_id,
+        },
+      });
+    await hook("sess-main", `agenthub-lane-${id}`);
+    assert.equal((await request("GET", `/api/lanes/${id}`)).body.lane.session_id, "sess-main");
+    // A split pane's session never takes the lane over…
+    await hook("sess-split", `agenthub-lane-${id}-2`);
+    assert.equal((await request("GET", `/api/lanes/${id}`)).body.lane.session_id, "sess-main");
+    // …while a session outside the dashboard still does, as before.
+    await hook("sess-outside", undefined);
+    assert.equal((await request("GET", `/api/lanes/${id}`)).body.lane.session_id, "sess-outside");
+  });
+
  it("clears needs_action only from the session that raised it, not from cross-session rebinds", async () => {
    const created = await request("POST", "/api/lanes", {
      cwd: "/tmp/lane-cross-session",
@@ -68,7 +68,13 @@ describe("pty-attach", () => {
      return "";
    });
    ptyAttach.attach(makeFakeWs(), "agenthub-lane-1", { cols: 80, rows: 24 });
-    assert.deepEqual(calls[0], ["set-option", "-t", "=agenthub-lane-1:", "mouse", "on"]);
+    assert.deepEqual(calls[0].slice(0, 5), [
+      "set-option",
+      "-t",
+      "=agenthub-lane-1:",
+      "mouse",
+      "on",
+    ]);
  });

  it("attaches with the RUNNING tmux server's own binary, not the one on PATH", () => {
@@ -1,8 +1,10 @@
 /**
 * @file Tests server/lib/read-pool.js: a read job runs in the worker on its own
 * connection, identical calls in flight share one run, a job over the timeout
- * is refused with EREADTIMEOUT and the next one gets a fresh worker, and
- * AGENTHUB_READ_WORKER=0 runs the job inline.
+ * is refused with EREADTIMEOUT and the next one gets a fresh worker,
+ * AGENTHUB_READ_WORKER=0 runs the job inline, `maxAgeMs` reuses a young result,
+ * the analytics events job matches the main connection, and the WAL checkpoint
+ * runs on the worker's writable connection.
 * @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
 */

@@ -63,6 +65,41 @@ describe("read-pool", () => {
    assert.equal(data.stats.totalSessions, 1);
  });

+  it("reuses a result younger than maxAgeMs, and only for the same args", async () => {
+    const first = await runRead(
+      "workflows",
+      { sections: ["stats"], tag: "memo" },
+      { maxAgeMs: 60_000 }
+    );
+    db.prepare("INSERT INTO sessions (id, status) VALUES ('rp-2', 'completed')").run();
+    const again = await runRead(
+      "workflows",
+      { sections: ["stats"], tag: "memo" },
+      { maxAgeMs: 60_000 }
+    );
+    assert.equal(again, first); // the cached object, not a new run
+    const fresh = await runRead("workflows", { sections: ["stats"], tag: "memo" });
+    assert.equal(fresh.stats.totalSessions, first.stats.totalSessions + 1);
+    db.prepare("DELETE FROM sessions WHERE id = 'rp-2'").run();
+  });
+
+  it("computes the analytics events numbers in the worker like the main connection", async () => {
+    const { eventAggregates } = require("../lib/scoped-stats");
+    const args = { scope: null, range: null, tzModifier: "+0 minutes" };
+    const inWorker = await runRead("analyticsEvents", args);
+    assert.deepEqual(inWorker, eventAggregates(db, null, null, "+0 minutes"));
+    assert.deepEqual(inWorker.toolUsage.map((r) => [r.tool_name, r.count]).sort(), [
+      ["Edit", 2],
+      ["Read", 2],
+    ]);
+  });
+
+  it("checkpoints the WAL from the worker", async () => {
+    const [row] = await runRead("walCheckpoint");
+    assert.equal(row.busy, 0);
+    assert.ok(row.log >= row.checkpointed);
+  });
+
  it("runs inline with AGENTHUB_READ_WORKER=0, and rejects an unknown job", async () => {
    process.env.AGENTHUB_READ_WORKER = "0";
    const data = await runRead("workflows", { sections: ["stats"], tag: "inline" });
@@ -401,8 +401,14 @@ describe("watchdog liveness reap", () => {
    const cwd = "/tmp/liveness-watchdog";
    await seedSession(sid, cwd);

-    liveness.probeLiveCwds = () => ({ available: true, cwds: new Set() });
-    hooksRouter.watchdogCheck();
+    // The watchdog's beat probes asynchronously (`ps` off the event loop).
+    const realAsync = liveness.probeLiveCwdsAsync;
+    liveness.probeLiveCwdsAsync = async () => ({ available: true, cwds: new Set() });
+    try {
+      await hooksRouter.watchdogCheck();
+    } finally {
+      liveness.probeLiveCwdsAsync = realAsync;
+    }

    assert.equal(stmts.getSession.get(sid).status, "completed");
  });
@@ -143,4 +143,26 @@ describe("subagent token backfill", () => {
    // a tokens key), so no session is re-scanned.
    assert.equal(res.sessions, 0);
  });
+
+  it("parses a row it can never stamp once, until its files change", async () => {
+    // A subagent row whose transcript holds no usage stays tokens-less forever,
+    // which used to re-parse its whole session tree on every boot.
+    stmts.insertAgent.run(
+      `${SESSION}-jsonl-ghost`,
+      SESSION,
+      "ghost",
+      "subagent",
+      "general-purpose",
+      "completed",
+      null,
+      MAIN,
+      JSON.stringify({ imported: true, source: "jsonl" })
+    );
+    fs.writeFileSync(path.join(tmpDir, SESSION, "subagents", "agent-ghost.jsonl"), "");
+    assert.equal((await importHistory.backfillSubagentTokenMetadata(dbModule)).sessions, 1);
+    assert.equal((await importHistory.backfillSubagentTokenMetadata(dbModule)).sessions, 0);
+    // A new subagent file appears: the session is looked at again.
+    fs.writeFileSync(path.join(tmpDir, SESSION, "subagents", "agent-new.jsonl"), "");
+    assert.equal((await importHistory.backfillSubagentTokenMetadata(dbModule)).sessions, 1);
+  });
 });
@@ -28,6 +28,35 @@ describe("tmux wrapper", () => {
    assert.equal(tmux.hasSession("agenthub-lane-1"), false);
  });

+  it("liveSessions answers every live name with ONE list-sessions call", () => {
+    const calls = [];
+    tmux.__setExecImpl((args) => {
+      calls.push(args[0]);
+      return "agenthub-lane-1\nagenthub-lane-12\nmain\n";
+    });
+    assert.deepEqual([...tmux.liveSessions()], ["agenthub-lane-1", "agenthub-lane-12", "main"]);
+    assert.deepEqual(calls, ["list-sessions"]);
+  });
+
+  it("liveSessions is empty when tmux answers that no server runs (or is not installed)", () => {
+    tmux.__setExecImpl(() => {
+      throw Object.assign(new Error("no server running"), { status: 1 });
+    });
+    assert.equal(tmux.liveSessions().size, 0);
+    tmux.__setExecImpl(() => {
+      throw Object.assign(new Error("spawn tmux ENOENT"), { code: "ENOENT" });
+    });
+    assert.equal(tmux.liveSessions().size, 0);
+  });
+
+  it("liveSessions is null when tmux could not be asked — not 'every pane is gone'", () => {
+    // Out of memory/processes: the panes are fine, the question failed.
+    tmux.__setExecImpl(() => {
+      throw Object.assign(new Error("spawnSync tmux EAGAIN"), { code: "EAGAIN" });
+    });
+    assert.equal(tmux.liveSessions(), null);
+  });
+
  it("newSession names a command that exited at once instead of tmux's set-option error", () => {
    tmux.__setExecImpl((args) => {
      if (args[0] === "new-session") return "";
@@ -79,6 +108,22 @@ describe("tmux wrapper", () => {
    calls = fake(true);
    tmux.enableMouse("agenthub-lane-1");
    assert.ok(calls.some((c) => c[0] === "set-option" && c[2] === "=agenthub-lane-1:"));
+    // tmux chains the four options into two spawns; psmux above got four.
+    const sets = calls.filter((c) => c[0] === "set-option");
+    assert.equal(sets.length, 2);
+    assert.deepEqual(sets[0], [
+      "set-option",
+      "-t",
+      "=agenthub-lane-1:",
+      "mouse",
+      "on",
+      ";",
+      "set-option",
+      "-t",
+      "=agenthub-lane-1:",
+      "status",
+      "off",
+    ]);
  });

  it("newSession builds the correct argv", () => {
@@ -104,16 +149,18 @@ describe("tmux wrapper", () => {
      "--model",
      "opus",
    ]);
-    assert.deepEqual(calls[1], ["set-option", "-t", "=agenthub-lane-1:", "mouse", "on"]);
-    assert.deepEqual(calls[2], ["set-option", "-t", "=agenthub-lane-1:", "status", "off"]);
-    assert.deepEqual(calls[3], ["set-option", "-t", "=agenthub-lane-1:", "pane-scrollbars", "on"]);
-    assert.deepEqual(calls[4], [
-      "set-option",
-      "-t",
-      "=agenthub-lane-1:",
-      "pane-scrollbars-style",
+    // One tmux call per `;`-chained group: the options every tmux knows, then
+    // the 3.7+ scrollbar pair (allowed to fail on its own).
+    assert.deepEqual(calls[1], [
+      ...["set-option", "-t", "=agenthub-lane-1:", "mouse", "on", ";"],
+      ...["set-option", "-t", "=agenthub-lane-1:", "status", "off"],
+    ]);
+    assert.deepEqual(calls[2], [
+      ...["set-option", "-t", "=agenthub-lane-1:", "pane-scrollbars", "on", ";"],
+      ...["set-option", "-t", "=agenthub-lane-1:", "pane-scrollbars-style"],
      "fg=brightwhite,bg=black,width=1,pad=0",
    ]);
+    assert.equal(calls.length, 3);
  });

  it("newSession still succeeds on a tmux older than 3.7 (no pane-scrollbars)", () => {
@@ -171,6 +218,22 @@ describe("tmux wrapper", () => {
    assert.deepEqual(tmux.listSessions("agenthub-lane-"), ["agenthub-lane-1", "agenthub-lane-2"]);
  });

+  it("shares one list-sessions answer for a second, until this process creates or kills one", () => {
+    const calls = [];
+    let names = "agenthub-lane-1\n";
+    tmux.__setExecImpl((args) => {
+      calls.push(args[0]);
+      return args[0] === "list-sessions" ? names : "";
+    });
+    assert.deepEqual(tmux.listSessions("agenthub-lane-"), ["agenthub-lane-1"]);
+    assert.deepEqual([...tmux.liveSessions()], ["agenthub-lane-1"]);
+    assert.equal(calls.filter((c) => c === "list-sessions").length, 1);
+    tmux.killSession("agenthub-lane-1");
+    names = "";
+    assert.deepEqual(tmux.listSessions("agenthub-lane-"), []);
+    assert.equal(calls.filter((c) => c === "list-sessions").length, 2);
+  });
+
  it("listSessions returns [] when tmux has no sessions at all (exit 1)", () => {
    tmux.__setExecImpl(() => {
      const e = new Error("no server running");