perf: v2.5.10 — ready for 10+ concurrent users, leaner locally

Measured on a copy of a real 1.2 GB / 253K-event database, 10 simulated browsers + 10 hooks/s for 90 s: event-loop delay p99 1239 ms → 32 ms (max 2.0 s → 94 ms), 266 failed requests → 0, 74 → 111 requests/s served, server RSS 633 → 346 MB. Entry bundle 975 KB raw → 394 KB (106 KB brotli). Server - /api/analytics: the events-table scans run in the read worker (scoped-stats eventAggregates, read-pool job analyticsEvents); runRead takes maxAgeMs — analytics 10 s, /api/workflows 30 s — keyed by the args - /api/workflows/skill-usage: +event_type and a tool_name range instead of LIKE, so it uses idx_events_tool (784 ms → a few ms) - idx_events_session_type_tool (session_id, event_type, tool_name, created_at) replaces idx_events_session_type: covering reads for session stats, the hook dedupes and the import resume point (1.2 s → 20 ms); 64 MB cache_size and 256 MB mmap_size on both connections - WAL: the server sets wal_autocheckpoint = 0 and checkpoints PASSIVE every DASHBOARD_WAL_CHECKPOINT_MS (5 s) from the read worker's own writable connection; a failed checkpoint restores the automatic one (hook COMMIT p95 109 ms → 15 ms at 100 hooks/s) - tmux: one list-sessions answers a whole GET /api/lanes and the heal beat (liveSessions), shared for 1 s and cleared by this process's new/kill — GET /api/run's sync spawns had blocked 60 s of a 90 s run; the heal beat skips tmux when nothing runs; attach options chained into 2 spawns - hooks: TurnDuration/APIError deduped against one read per hook instead of a query per transcript entry (23 ms/hook on a 763-turn session); one lane scan per hook; hourly upserts skip rows that did not grow; frames are sent after the transaction commits (and after the pane binding); new_event carries its row id; a split pane's (slot 1–9) hooks no longer rebind the lane's session_id - watchdog liveness probe runs ps asynchronously; transcript newline scan via indexOf; startup subagent backfill fingerprints the files it already checked - pty attach: backpressure (pause past 1 MB unsent, resume under 256 KB) - static: hashed assets served from their pre-compressed .br/.gz sibling Client - hidden tabs neither refetch on WS events nor poll (useLiveRefresh, panes, lane cards, git header, changes panel, settings); no per-tab hourly /updates/check (the server already checks) - refetch storms throttled (workspace tree/session list 3 s, /api/run only on a run-key change, notification pane lookup only when shown); WS reconnect jitter; terminal retry backoff 1.5 → 30 s, stops on 1008 - heavier throttles for aggregates (15 s) and detail views; Monitor keepIfSame; GET in-flight dedupe in request() - lazy Dashboard/History, xterm (with the first terminal), lane modals, Sagi (only while enabled); only the active locale loads; Workspace chunk prefetched on a /workspace deep link - Sagi: no work while disabled, pupil via CSS variables, memo avatar - build: .br/.gz siblings, sourcemap hidden; service worker no longer caches /assets/ (cache v3 drops the old one) Tests: scripts/test-server.js drops the GIT_DIR-family variables a hook exports, so a suite run from a linked worktree's pre-commit can no longer commit into the real repository or set core.bare. Docs: ARCHITECTURE, DATABASE, API, DEPLOYMENT, client README, docs/plans/perf/README.md (audit, checklist, results).
2026-10-07 13:24:15 +07:00
parent 8fdda66893
commit ca821008e2
75 changed files with 1711 additions and 699 deletions
@@ -12,8 +12,8 @@ const TranscriptCache = require("./transcript-cache");

 /** Write every hour of one transcript parse (MAX per hour, see the statement). */
 function writeHourly(sessionId, tokensByHour) {
-  // ponytail: rewrites every hour of the session on each hook — ~1 row per
-  // active hour per model, cheap; write only the newest hours if it ever shows.
+  // Runs every hour of the session on each hook — ~1 row per active hour per
+  // model; the statement leaves a row that did not grow untouched (no write).
  for (const t of Object.values(tokensByHour || {})) {
    stmts.upsertHourlyUsage.run(
      sessionId,
@@ -601,14 +601,15 @@ function recoverInterruptedProvisioning() {
 * and a `codex` lane (see `idx_lanes_cwd_tool`). Omitted, it matches any tool,
 * which stays correct for callers that already know they hold the lane's own
 * cwd (there's only ever one match for that exact path per tool, and this
- * caller shape doesn't care which).
+ * caller shape doesn't care which). `lanes` lets a caller that resolves several
+ * paths (a hook) read the lane table once.
 */
-function resolveLaneByCwd(cwd, tool) {
+function resolveLaneByCwd(cwd, tool, lanes = null) {
  if (!cwd || typeof cwd !== "string") return null;
  // A Windows cwd ends in and nests under `\`.
  const target = cwd.replace(/[\\/]+$/, "");
  let best = null;
-  for (const lane of listLanes()) {
+  for (const lane of lanes || listLanes()) {
    if (tool && lane.tool !== tool) continue;
    const base = lane.cwd.replace(/[\\/]+$/, "");
    if (target === base || target.startsWith(`${base}/`) || target.startsWith(base + path.sep)) {
@@ -654,13 +655,14 @@ function absolutePathsIn(toolInput) {
 * has no equivalent hook mechanism here) — so both lookups are pinned to the
 * `claude` lane at each path, never the `codex` one that might share it.
 */
-function resolveLaneForWork(sessionCwd, toolInput) {
+function resolveLaneForWork(sessionCwd, toolInput, lanes = null) {
+  const all = lanes || listLanes();
  let best = null;
  for (const candidate of absolutePathsIn(toolInput)) {
-    const lane = resolveLaneByCwd(candidate, "claude");
+    const lane = resolveLaneByCwd(candidate, "claude", all);
    if (lane && (!best || lane.cwd.length > best.cwd.length)) best = lane;
  }
-  return best || resolveLaneByCwd(sessionCwd, "claude");
+  return best || resolveLaneByCwd(sessionCwd, "claude", all);
 }

 function classifyLiveness({ status, stage, ageSec }, deadSec = DEAD_SEC) {
@@ -81,9 +81,24 @@ function attach(ws, runId, { cols, rows }) {
    rows: rows || 24,
  });

+  // Backpressure: a slow viewer (a remote browser on a thin link) must not
+  // queue a chatty pane's output in this process without bound. Past 1 MB
+  // unsent the pty is paused — tmux holds the pane and redraws — and resumed
+  // once the socket drains below 256 KB.
+  let paused = false;
+  const drained = () => {
+    if (paused && ws.bufferedAmount < 256 * 1024) {
+      paused = false;
+      pty.resume?.();
+    }
+  };
  pty.onData((data) => {
    try {
-      ws.send(data, { binary: true });
+      ws.send(data, { binary: true }, drained);
+      if (!paused && ws.bufferedAmount > 1024 * 1024) {
+        paused = true;
+        pty.pause?.();
+      }
    } catch {
      /* client gone between data event and send — safe to ignore */
    }
@@ -2,10 +2,11 @@
 * @file Runs the dashboard's heavy read-only aggregations in a worker thread
 * on its own SQLite connection, so a slow one never blocks the event loop that
 * serves every other page, hook and websocket. Identical requests in flight are
- * shared, a job over `AGENTHUB_READ_TIMEOUT_MS` (default 60 s) is answered with
- * `EREADTIMEOUT` and its worker replaced, and `AGENTHUB_READ_WORKER=0` — or a
- * worker that cannot start — runs the job inline as before. This file is also
- * the worker's entry point.
+ * shared (and, when the caller asks, reused for a short window), a job over
+ * `AGENTHUB_READ_TIMEOUT_MS` (default 60 s) is answered with `EREADTIMEOUT` and
+ * its worker replaced, and `AGENTHUB_READ_WORKER=0` — or a worker that cannot
+ * start — runs the job inline as before. The server's periodic WAL checkpoint
+ * also runs here (`walCheckpoint`). This file is also the worker's entry point.
 * @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
 */

@@ -19,6 +20,11 @@ const JOBS = {
      { status: a.status || null, tool: a.tool || null, range: a.range || null },
      a.sections || null
    ),
+  // Not a read: the worker runs it on its own writable connection (below), so
+  // the checkpoint's copy into the database file never blocks the event loop.
+  walCheckpoint: (conn) => conn.pragma("wal_checkpoint(PASSIVE)", { simple: false }),
+  analyticsEvents: (conn, a) =>
+    require("./scoped-stats").eventAggregates(conn, a.scope, a.range, a.tzModifier),
 };

 function timeoutMs() {
@@ -35,6 +41,7 @@ let worker = null;
 let seq = 0;
 const pending = new Map(); // id → { resolve, reject, timer, w }
 const inflight = new Map(); // job+args → Promise
+const recent = new Map(); // job+args → { at, result } for callers passing maxAgeMs

 function spawn(dbPath) {
  const w = new Worker(__filename, { workerData: { agenthubReadWorker: true, dbPath } });
@@ -93,37 +100,59 @@ function dispatch(job, args) {

 /**
 * Run a named read job off the event loop. Concurrent calls with the same job
- * and args share one run.
+ * and args share one run; with `maxAgeMs`, a result that young is reused too,
+ * so N open dashboards cost one query per window instead of N. Args are part
+ * of the key, so a result is only reused for the very same query.
 *
 * @param {keyof JOBS} job
 * @param {object} [args] structured-cloneable
+ * @param {{maxAgeMs?: number}} [opts]
 * @returns {Promise<any>}
 */
-function runRead(job, args = {}) {
+function runRead(job, args = {}, { maxAgeMs = 0 } = {}) {
  if (!JOBS[job]) return Promise.reject(codedError(`unknown read job ${job}`, "EBADJOB"));
  const key = `${job}:${JSON.stringify(args)}`;
+  const now = Date.now();
+  const fresh = recent.get(key);
+  if (fresh && now - fresh.at < maxAgeMs) return Promise.resolve(fresh.result);
  const hit = inflight.get(key);
  if (hit) return hit;
-  const p = dispatch(job, args).finally(() => inflight.delete(key));
+  const p = dispatch(job, args)
+    .then((result) => {
+      if (maxAgeMs > 0) {
+        // ponytail: drop-expired-on-write keeps this bounded by the keys used
+        // within one window; an LRU if a caller ever varies args per request.
+        for (const [k, v] of recent) if (now - v.at > 60_000) recent.delete(k);
+        recent.set(key, { at: Date.now(), result });
+      }
+      return result;
+    })
+    .finally(() => inflight.delete(key));
  inflight.set(key, p);
  return p;
 }

 if (!isMainThread && workerData?.agenthubReadWorker) {
  // Same driver order as server/db.js: better-sqlite3, else node:sqlite.
-  let conn;
-  try {
-    const Database = require("better-sqlite3");
-    conn = new Database(workerData.dbPath);
-  } catch {
-    const Database = require("../compat-sqlite");
-    conn = new Database(workerData.dbPath);
-  }
+  const open = () => {
+    try {
+      const Database = require("better-sqlite3");
+      return new Database(workerData.dbPath);
+    } catch {
+      const Database = require("../compat-sqlite");
+      return new Database(workerData.dbPath);
+    }
+  };
+  const conn = open();
+  let writer = null; // walCheckpoint's connection, opened on first use
  conn.pragma("busy_timeout = 5000");
  conn.pragma("query_only = ON"); // this connection can never write
+  conn.pragma("cache_size = -65536"); // same read tuning as server/db.js
+  conn.pragma("mmap_size = 268435456");
  parentPort.on("message", ({ id, job, args }) => {
    try {
-      parentPort.postMessage({ id, result: JOBS[job](conn, args) });
+      const c = job === "walCheckpoint" ? (writer ??= open()) : conn;
+      parentPort.postMessage({ id, result: JOBS[job](c, args) });
    } catch (err) {
      parentPort.postMessage({ id, error: { message: err.message, code: err.code } });
    }
@@ -197,7 +197,49 @@ function avgEventsPerSession(db, scope, range) {
    .get(...scope.params, ...s.params, ...scope.params, ...e.params);
 }

+/**
+ * The analytics numbers that scan `events` (tool usage, daily events, event
+ * types, events per session) for a scope or none. Takes the connection as an
+ * argument so the read worker can run it (`analyticsEvents` in read-pool.js).
+ */
+function eventAggregates(db, scope, range, tzModifier) {
+  if (scope) {
+    return {
+      toolUsage: toolUsageCounts(db, scope, range),
+      dailyEvents: dailyEventCounts(db, scope, tzModifier),
+      eventTypes: eventTypeCounts(db, scope, range),
+      avgEvents: avgEventsPerSession(db, scope, range),
+    };
+  }
+  return {
+    toolUsage: db
+      .prepare(
+        `SELECT tool_name, COUNT(*) as count FROM events WHERE tool_name IS NOT NULL
+         GROUP BY tool_name ORDER BY count DESC LIMIT 20`
+      )
+      .all(),
+    dailyEvents: db
+      .prepare(
+        `SELECT DATE(created_at, ?) as date, COUNT(*) as count FROM events
+         WHERE created_at >= DATE('now', '-365 days') GROUP BY 1 ORDER BY date ASC`
+      )
+      .all(tzModifier),
+    eventTypes: db
+      .prepare(
+        "SELECT event_type, COUNT(*) as count FROM events GROUP BY event_type ORDER BY count DESC"
+      )
+      .all(),
+    avgEvents: db
+      .prepare(
+        `SELECT ROUND(CAST(COUNT(*) AS REAL) / MAX(1, (SELECT COUNT(*) FROM sessions)), 1) as avg
+         FROM events`
+      )
+      .get(),
+  };
+}
+
 module.exports = {
+  eventAggregates,
  statsOverview,
  agentStatusCounts,
  sessionStatusCounts,
@@ -16,7 +16,7 @@
 * @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
 */

-const { execFileSync } = require("node:child_process");
+const { execFile, execFileSync } = require("node:child_process");
 const fs = require("node:fs");
 const path = require("node:path");
 const { isInsideContainer } = require("../../scripts/install-hooks");
@@ -63,21 +63,45 @@ function probeDisabledByEnv() {
 * with an empty set genuinely means no claude process is running.
 */
 function probeLiveCwds() {
-  if (probeDisabledByEnv()) return UNAVAILABLE();
-  if (process.platform === "win32") return UNAVAILABLE();
-  if (isInsideContainer()) return UNAVAILABLE();
-
+  if (probeBlocked()) return UNAVAILABLE();
  let psOut;
  try {
-    psOut = execImpl("ps", ["-Ao", "pid=,args="], {
-      encoding: "utf8",
-      timeout: 5_000,
-      maxBuffer: 16 * 1024 * 1024,
-    });
+    psOut = execImpl("ps", PS_ARGS, PS_OPTS);
  } catch {
    return UNAVAILABLE();
  }
+  return cwdsFromPs(psOut);
+}

+/**
+ * `probeLiveCwds` with `ps` run asynchronously — what the watchdog's 15 s
+ * beat uses, so the spawn (24–300 ms inside a large server process) never
+ * blocks the event loop. Same answer shape; resolves, never rejects.
+ *
+ * @returns {Promise<{ available: boolean, cwds: Set<string> }>}
+ */
+function probeLiveCwdsAsync() {
+  if (probeBlocked()) return Promise.resolve(UNAVAILABLE());
+  return new Promise((resolve) => {
+    execFile("ps", PS_ARGS, PS_OPTS, (err, out) => {
+      try {
+        resolve(err ? UNAVAILABLE() : cwdsFromPs(out));
+      } catch {
+        resolve(UNAVAILABLE());
+      }
+    });
+  });
+}
+
+const PS_ARGS = ["-Ao", "pid=,args="];
+const PS_OPTS = { encoding: "utf8", timeout: 5_000, maxBuffer: 16 * 1024 * 1024 };
+
+function probeBlocked() {
+  return probeDisabledByEnv() || process.platform === "win32" || isInsideContainer();
+}
+
+/** The cwds of the `claude` processes in a `ps -Ao pid=,args=` listing. */
+function cwdsFromPs(psOut) {
  const pids = [];
  for (const line of psOut.split("\n")) {
    const m = line.match(/^\s*(\d+)\s+(.*)$/);
@@ -120,4 +144,4 @@ function probeLiveCwds() {
  return { available: true, cwds };
 }

-module.exports = { probeLiveCwds, isClaudeCommand, __setExecImpl };
+module.exports = { probeLiveCwds, probeLiveCwdsAsync, isClaudeCommand, __setExecImpl };
@@ -34,10 +34,35 @@ let exactTargets = null;
 function __setExecImpl(fn, { exact = true } = {}) {
  execImpl = fn;
  exactTargets = exact;
+  listed = null;
 }
 function __reset() {
  execImpl = (args) => execFileSync("tmux", args, EXEC_OPTS);
  exactTargets = null;
+  listed = null;
+}
+
+// The `list-sessions` answer, shared for LIST_TTL_MS. Every open dashboard
+// polls GET /api/run, and each synchronous spawn forks this whole process —
+// measured 2,247 calls, 60 s of a 90 s run blocked, at 10 browsers. A session
+// this process creates or kills clears it, so only a pane that exits on its
+// own can read as live for up to a second. "No server" (exit 1) is an answer
+// too and is shared; any other failure is not.
+const LIST_TTL_MS = 1000;
+let listed = null; // { at, out } | { at, err }
+function listOutput() {
+  if (listed && Date.now() - listed.at < LIST_TTL_MS) {
+    if (listed.err) throw listed.err;
+    return listed.out;
+  }
+  try {
+    const out = String(execImpl(["list-sessions", "-F", "#{session_name}"]));
+    listed = { at: Date.now(), out };
+    return out;
+  } catch (err) {
+    listed = err.status === 1 || err.code === "ENOENT" ? { at: Date.now(), err } : null;
+    throw err;
+  }
 }

 /**
@@ -117,6 +142,7 @@ function envFlags(env) {
 function newSession({ name, cwd, argv, env }) {
  // No argv = tmux's own default shell (a Windows Terminal pane).
  const cmd = argv.length ? ["--", ...argv] : [];
+  listed = null;
  execImpl(["new-session", "-d", "-s", name, "-c", cwd, ...envFlags(env), ...cmd]);
  try {
    enableMouse(name);
@@ -144,25 +170,37 @@ function newSession({ name, cwd, argv, env }) {
 // The status bar goes off here too: the dashboard's tab bar already names the
 // pane, and tmux's green line only eats a row of the terminal.
 function enableMouse(name) {
-  execImpl(["set-option", "-t", pane(name), "mouse", "on"]);
-  execImpl(["set-option", "-t", pane(name), "status", "off"]);
+  // tmux runs `;`-separated commands from ONE argv (each tmux call is a
+  // synchronous spawn on the event loop), stopping at the first that fails —
+  // so the options every tmux knows go first, in their own call. psmux (no
+  // `=name` targets) is not known to chain, so it gets one call per option.
+  const run = (...cmds) => {
+    if (exact()) return execImpl(cmds.flatMap((c, i) => (i ? [";", ...c] : c)));
+    for (const c of cmds) execImpl(c);
+  };
+  run(
+    ["set-option", "-t", pane(name), "mouse", "on"],
+    ["set-option", "-t", pane(name), "status", "off"]
+  );
  // A draggable scrollbar beside the pane (mouse mode makes it drag). Only a
  // tmux SERVER of 3.7+ knows the option; an older one refuses it, which is
  // fine — the pane just has none (`/agenthub-doctor` says how to upgrade).
  try {
-    execImpl(["set-option", "-t", pane(name), "pane-scrollbars", "on"]);
-    // tmux's default (`fg=white,bg=black,width=1`) is a one-column track in
-    // ANSI black — the terminal theme's border colour, almost the background —
-    // so it reads as no scrollbar at all. Bright slider. ONE column: tmux 3.7c
-    // reserves `width` columns but paints only the first, so a second one is
-    // never cleared and shows stale cells — a ghost second scrollbar.
-    execImpl([
-      "set-option",
-      "-t",
-      pane(name),
-      "pane-scrollbars-style",
-      "fg=brightwhite,bg=black,width=1,pad=0",
-    ]);
+    run(
+      ["set-option", "-t", pane(name), "pane-scrollbars", "on"],
+      // tmux's default (`fg=white,bg=black,width=1`) is a one-column track in
+      // ANSI black — the terminal theme's border colour, almost the background —
+      // so it reads as no scrollbar at all. Bright slider. ONE column: tmux 3.7c
+      // reserves `width` columns but paints only the first, so a second one is
+      // never cleared and shows stale cells — a ghost second scrollbar.
+      [
+        "set-option",
+        "-t",
+        pane(name),
+        "pane-scrollbars-style",
+        "fg=brightwhite,bg=black,width=1,pad=0",
+      ]
+    );
  } catch {
    /* tmux < 3.7 */
  }
@@ -239,6 +277,7 @@ function pasteText(name, text) {

 /** Idempotent — a session that's already gone is not an error. */
 function killSession(name) {
+  listed = null;
  try {
    execImpl(["kill-session", "-t", session(name)]);
  } catch {
@@ -250,7 +289,7 @@ function killSession(name) {
 function listSessions(prefix) {
  let out;
  try {
-    out = execImpl(["list-sessions", "-F", "#{session_name}"]);
+    out = listOutput();
  } catch {
    return [];
  }
@@ -260,6 +299,27 @@ function listSessions(prefix) {
    .filter((s) => s && s.startsWith(prefix));
 }

+/**
+ * Every live session name from ONE `list-sessions` — what a poll asks instead
+ * of a synchronous has-session per name, each of which blocks the event loop
+ * (and so every terminal keystroke) for a whole tmux spawn. Empty when no
+ * server runs (tmux exits 1) or tmux is not installed; null when the call
+ * itself failed (EAGAIN/ENOMEM under memory pressure), which proves nothing
+ * about the panes — a caller must not read that as "every pane is gone".
+ */
+function liveSessions() {
+  try {
+    return new Set(
+      listOutput()
+        .split("\n")
+        .map((s) => s.trim())
+        .filter(Boolean)
+    );
+  } catch (err) {
+    return err.status === 1 || err.code === "ENOENT" ? new Set() : null;
+  }
+}
+
 function isTmuxAvailable() {
  return clientVersion() !== null;
 }
@@ -297,6 +357,7 @@ module.exports = {
  pasteText,
  killSession,
  listSessions,
+  liveSessions,
  isTmuxAvailable,
  clientVersion,
  serverVersion,
@@ -312,9 +312,10 @@ class TranscriptCache {
        pos += got;

        let lineStart = 0;
-        for (let i = 0; i < got; i++) {
-          if (buf[i] !== 0x0a) continue;
-
+        // indexOf finds each newline natively — a JS byte loop was ~100 ms of
+        // a 76 MB transcript's cold read on the event loop.
+        const view = buf.subarray(0, got);
+        for (let i = view.indexOf(0x0a); i !== -1; i = view.indexOf(0x0a, i + 1)) {
          let line;
          if (pendingLen) {
            const need = pendingLen + (i - lineStart);