perf: v2.5.10 — ready for 10+ concurrent users, leaner locally

Measured on a copy of a real 1.2 GB / 253K-event database, 10 simulated
browsers + 10 hooks/s for 90 s: event-loop delay p99 1239 ms → 32 ms (max
2.0 s → 94 ms), 266 failed requests → 0, 74 → 111 requests/s served, server
RSS 633 → 346 MB. Entry bundle 975 KB raw → 394 KB (106 KB brotli).

Server
- /api/analytics: the events-table scans run in the read worker
  (scoped-stats eventAggregates, read-pool job analyticsEvents); runRead takes
  maxAgeMs — analytics 10 s, /api/workflows 30 s — keyed by the args
- /api/workflows/skill-usage: +event_type and a tool_name range instead of
  LIKE, so it uses idx_events_tool (784 ms → a few ms)
- idx_events_session_type_tool (session_id, event_type, tool_name,
  created_at) replaces idx_events_session_type: covering reads for session
  stats, the hook dedupes and the import resume point (1.2 s → 20 ms); 64 MB
  cache_size and 256 MB mmap_size on both connections
- WAL: the server sets wal_autocheckpoint = 0 and checkpoints PASSIVE every
  DASHBOARD_WAL_CHECKPOINT_MS (5 s) from the read worker's own writable
  connection; a failed checkpoint restores the automatic one (hook COMMIT p95
  109 ms → 15 ms at 100 hooks/s)
- tmux: one list-sessions answers a whole GET /api/lanes and the heal beat
  (liveSessions), shared for 1 s and cleared by this process's new/kill —
  GET /api/run's sync spawns had blocked 60 s of a 90 s run; the heal beat
  skips tmux when nothing runs; attach options chained into 2 spawns
- hooks: TurnDuration/APIError deduped against one read per hook instead of a
  query per transcript entry (23 ms/hook on a 763-turn session); one lane
  scan per hook; hourly upserts skip rows that did not grow; frames are sent
  after the transaction commits (and after the pane binding); new_event
  carries its row id; a split pane's (slot 1–9) hooks no longer rebind the
  lane's session_id
- watchdog liveness probe runs ps asynchronously; transcript newline scan via
  indexOf; startup subagent backfill fingerprints the files it already checked
- pty attach: backpressure (pause past 1 MB unsent, resume under 256 KB)
- static: hashed assets served from their pre-compressed .br/.gz sibling

Client
- hidden tabs neither refetch on WS events nor poll (useLiveRefresh, panes,
  lane cards, git header, changes panel, settings); no per-tab hourly
  /updates/check (the server already checks)
- refetch storms throttled (workspace tree/session list 3 s, /api/run only on
  a run-key change, notification pane lookup only when shown); WS reconnect
  jitter; terminal retry backoff 1.5 → 30 s, stops on 1008
- heavier throttles for aggregates (15 s) and detail views; Monitor keepIfSame;
  GET in-flight dedupe in request()
- lazy Dashboard/History, xterm (with the first terminal), lane modals, Sagi
  (only while enabled); only the active locale loads; Workspace chunk
  prefetched on a /workspace deep link
- Sagi: no work while disabled, pupil via CSS variables, memo avatar
- build: .br/.gz siblings, sourcemap hidden; service worker no longer caches
  /assets/ (cache v3 drops the old one)

Tests: scripts/test-server.js drops the GIT_DIR-family variables a hook
exports, so a suite run from a linked worktree's pre-commit can no longer
commit into the real repository or set core.bare.

Docs: ARCHITECTURE, DATABASE, API, DEPLOYMENT, client README,
docs/plans/perf/README.md (audit, checklist, results).

server/routes/analytics.js

@@ -14,6 +14,7 @@ const {
   BAD_RANGE,
 } = require("../lib/source-filter");
 const scoped = require("../lib/scoped-stats");
+const { runRead } = require("../lib/read-pool");
 
 const { calculateCost, dailyTokenRows, costWhere } = require("./pricing");
 
@@ -34,12 +35,12 @@ const USAGE_TOKEN_COLS = {
  * With a range, counts follow it and tokens/cost come from the cost rows of the
  * days in it (`token_usage` has no date); the daily charts keep 365 days.
  */
-function computeAnalytics({ scope = null, range = null, tzModifier = "+0 minutes" } = {}) {
+function computeAnalytics(
+  { scope = null, range = null, tzModifier = "+0 minutes" } = {},
+  events = scoped.eventAggregates(db, scope || (range ? ALL : null), range, tzModifier)
+) {
   const sc = scope || (range ? ALL : null);
-  const toolUsage = sc ? scoped.toolUsageCounts(db, sc, range) : stmts.toolUsageCounts.all();
-  const dailyEvents = sc
-    ? scoped.dailyEventCounts(db, sc, tzModifier)
-    : stmts.dailyEventCounts.all(tzModifier);
+  const { toolUsage, dailyEvents, eventTypes, avgEvents } = events;
   const dailySessions = sc
     ? scoped.dailySessionCounts(db, sc, tzModifier)
     : stmts.dailySessionCounts.all(tzModifier);
@@ -56,10 +57,6 @@ function computeAnalytics({ scope = null, range = null, tzModifier = "+0 minutes
   const totalSubagents = sc
     ? scoped.totalSubagentCount(db, sc, range)
     : stmts.totalSubagentCount.get();
-  const eventTypes = sc ? scoped.eventTypeCounts(db, sc, range) : stmts.eventTypeCounts.all();
-  const avgEvents = sc
-    ? scoped.avgEventsPerSession(db, sc, range)
-    : stmts.avgEventsPerSession.get();
 
   // Total cost exactly as GET /api/pricing/cost computes it (baselines, the
   // per-day rate, the same scope and range), so the two numbers always agree.
@@ -108,7 +105,20 @@ router.get("/", (req, res) => {
   if (parseTool(req) === false) return res.status(400).json(BAD_TOOL);
   const range = parseRange(req);
   if (range === false) return res.status(400).json(BAD_RANGE);
-  res.json(computeAnalytics({ scope: parseScope(req), range, tzModifier }));
+  const scope = parseScope(req);
+  // The events scans run in the read worker (a 365-day daily count is ~0.5 s
+  // on a large DB), and a 10 s memo lets every open dashboard share one run.
+  runRead(
+    "analyticsEvents",
+    { scope: scope || (range ? ALL : null), range, tzModifier },
+    { maxAgeMs: 10_000 }
+  )
+    .then((events) => res.json(computeAnalytics({ scope, range, tzModifier }, events)))
+    .catch((err) =>
+      res
+        .status(err.code === "EREADTIMEOUT" ? 503 : 500)
+        .json({ error: { code: err.code, message: err.message } })
+    );
 });
 
 module.exports = router;

server/routes/hooks.js

@@ -7,7 +7,15 @@ const { Router } = require("express");
 const { v4: uuidv4 } = require("uuid");
 const dbModule = require("../db");
 const { stmts, db } = dbModule;
-const { broadcast } = require("../websocket");
+const { broadcast: sendNow } = require("../websocket");
+
+// Frames emitted while a hook's transaction is open wait in `held` for its
+// commit (see processEvent); everywhere else they go out at once.
+let held = null;
+function broadcast(type, data) {
+  if (held) held.push([type, data]);
+  else sendNow(type, data);
+}
 const TranscriptCache = require("../lib/transcript-cache");
 const { writeHourly } = require("../lib/hourly-usage");
 const { scanAndImportSubagents } = require("../../scripts/import-history");
@@ -173,13 +181,16 @@ const LANE_DONE_HOOKS = new Set(["Stop", "SessionEnd"]);
  * running/idle, so without this an adopted lane read "idle" for the entire
  * time Claude was working in it.
  */
-function touchLaneFromHook(hookType, data, tool = "claude") {
+function touchLaneFromHook(hookType, data, tool = "claude", runId = null) {
   try {
     if (!data || !data.cwd) return;
+    // One read of the lane table for every path this hook resolves.
+    // ponytail: a scan per hook — fine at dozens of lanes; index by cwd prefix if lane counts grow.
+    const allLanes = lanesLib.listLanes();
     // Pin to the lane of the CLI that sent the hook — a Claude hook never
     // touches the codex lane sharing its directory, nor a Codex hook the
     // claude one.
-    const lane = lanesLib.resolveLaneByCwd(data.cwd, tool);
+    const lane = lanesLib.resolveLaneByCwd(data.cwd, tool, allLanes);
     if (!lane) return;
 
     // Infer the lane's stage from this one tool event, best-effort. A real
@@ -203,7 +214,7 @@ function touchLaneFromHook(hookType, data, tool = "claude") {
         // Credit the lane whose files/directory the tool actually touched, which
         // is not always the session's lane - see resolveLaneForWork. Bookkeeping
         // below still uses `lane`, the session's own.
-        const workLane = lanesLib.resolveLaneForWork(data.cwd, data.tool_input) || lane;
+        const workLane = lanesLib.resolveLaneForWork(data.cwd, data.tool_input, allLanes) || lane;
         const detection = detect(getPipeline(workLane.pipeline), {
           tool_name: data.tool_name,
           tool_input: data.tool_input,
@@ -222,7 +233,14 @@ function touchLaneFromHook(hookType, data, tool = "claude") {
     // resuming, so it neither clears needs_action nor moves the lane.
     const childEvent = tool === "codex" && data._codexChild === true;
     const idleNudge = hookType === "Notification" && IDLE_NUDGE_RE.test(data.message || "");
-    if (data.session_id && lane.session_id !== data.session_id) patch.session_id = data.session_id;
+    // The lane's session follows its slot-0 pane (or a session outside the
+    // dashboard), never a split pane's: only slot 0 is the lane's run, and two
+    // panes of one workspace would otherwise flip session_id — and broadcast
+    // the lane to every browser — on every hook.
+    const splitPane = typeof runId === "string" && /^agenthub-lane-\d+-[1-9]$/.test(runId);
+    if (data.session_id && lane.session_id !== data.session_id && !splitPane) {
+      patch.session_id = data.session_id;
+    }
     if (hookType === "Notification" && !idleNudge) {
       patch.needs_action = data.message || "needs you";
       patch.needs_action_session = data.session_id || null;
@@ -260,6 +278,14 @@ function touchLaneFromHook(hookType, data, tool = "claude") {
 }
 
 const setSessionTool = db.prepare("UPDATE sessions SET tool = ? WHERE id = ?");
+// What processEvent dedupes the transcript's API errors and turn durations
+// against — one read per hook, not one per transcript entry.
+const apiErrorSummariesStmt = db.prepare(
+  "SELECT summary FROM events WHERE session_id = ? AND event_type = 'APIError'"
+);
+const turnDurationStampsStmt = db.prepare(
+  "SELECT created_at FROM events WHERE session_id = ? AND event_type = 'TurnDuration'"
+);
 
 /** The session's `<id>-main` agent, created and broadcast. */
 function createMainAgent(session) {
@@ -448,7 +474,7 @@ function applyFirstUserDescriptor(sessionId, result) {
   if (refreshedAgent) broadcast("agent_updated", refreshedAgent);
 }
 
-const processEvent = db.transaction((hookType, data, tool = "claude") => {
+const processEventTx = db.transaction((hookType, data, tool = "claude") => {
   const sessionId = data.session_id;
   if (!sessionId) return null;
 
@@ -1017,16 +1043,13 @@ const processEvent = db.transaction((hookType, data, tool = "claude") => {
       // Register API errors from transcript (quota limits, rate limits, overloaded, etc.)
       if (result.errors) {
         let newErrorRecorded = false;
+        // Deduplicate against the errors already recorded, read ONCE per hook:
+        // `result.errors` is the transcript's whole history, not just new ones.
+        const recorded = new Set(apiErrorSummariesStmt.all(sessionId).map((r) => r.summary));
         for (const apiErr of result.errors) {
-          // Deduplicate: check if we already recorded this error (same type+message+timestamp)
-          const errKey = `${apiErr.type}:${apiErr.timestamp || ""}`;
-          const existing = db
-            .prepare(
-              `SELECT 1 FROM events WHERE session_id = ? AND event_type = 'APIError'
-               AND summary = ? LIMIT 1`
-            )
-            .get(sessionId, `${apiErr.type}: ${apiErr.message}`);
-          if (existing) continue;
+          const errSummary = `${apiErr.type}: ${apiErr.message}`;
+          if (recorded.has(errSummary)) continue;
+          recorded.add(errSummary);
 
           stmts.insertEvent.run(
             sessionId,
@@ -1096,15 +1119,15 @@ const processEvent = db.transaction((hookType, data, tool = "claude") => {
 
       // Register turn duration events from transcript
       if (result.turnDurations) {
+        // `turnDurations` is every turn in the transcript (up to 1000), on
+        // every hook: dedupe against ONE read of the recorded timestamps (a
+        // covering-index scan) instead of a query per turn — 23 ms per hook on
+        // a 763-turn session before.
+        const recorded = new Set(turnDurationStampsStmt.all(sessionId).map((r) => r.created_at));
         for (const td of result.turnDurations) {
           const tdTs = td.timestamp || new Date().toISOString();
-          // Deduplicate by checking if we already have this turn duration event
-          const existing = db
-            .prepare(
-              "SELECT 1 FROM events WHERE session_id = ? AND event_type = 'TurnDuration' AND created_at = ? LIMIT 1"
-            )
-            .get(sessionId, tdTs);
-          if (existing) continue;
+          if (recorded.has(tdTs)) continue;
+          recorded.add(tdTs);
 
           const tdSummary = `Turn completed in ${(td.durationMs / 1000).toFixed(1)}s`;
           // Stamped with the TRANSCRIPT's timestamp, not NOW — that is what the
@@ -1168,7 +1191,7 @@ const processEvent = db.transaction((hookType, data, tool = "claude") => {
   // Bump session updated_at on every event
   stmts.touchSession.run(sessionId);
 
-  stmts.insertEvent.run(
+  const inserted = stmts.insertEvent.run(
     sessionId,
     agentId,
     eventType,
@@ -1179,6 +1202,8 @@ const processEvent = db.transaction((hookType, data, tool = "claude") => {
   );
 
   const event = {
+    // The row id (additive): lets a client drop a frame it already holds.
+    id: Number(inserted.lastInsertRowid),
     session_id: sessionId,
     agent_id: agentId,
     event_type: eventType,
@@ -1190,6 +1215,31 @@ const processEvent = db.transaction((hookType, data, tool = "claude") => {
   return event;
 });
 
+/**
+ * One hook's writes in one transaction, its WebSocket frames sent only once
+ * that transaction COMMITTED — in order, in the same tick. A frame sent from
+ * inside it described a row a later throw could still roll back.
+ */
+function processEvent(hookType, data, tool) {
+  return holdFrames(() => processEventTx(hookType, data, tool));
+}
+
+/** Run `fn` with frames held, then send them if it returned (nested: the
+ *  outermost call sends). A throw drops them. */
+function holdFrames(fn) {
+  if (held) return fn();
+  held = [];
+  let frames = null;
+  try {
+    const out = fn();
+    frames = held;
+    return out;
+  } finally {
+    held = null;
+    if (frames) for (const [type, payload] of frames) sendNow(type, payload);
+  }
+}
+
 // A Codex sub-agent thread's own lifecycle is not the user's session's: its
 // Stop is not the end of the user's turn, nor its SessionStart a new session.
 const CODEX_CHILD_DROPPED = new Set([
@@ -1414,26 +1464,38 @@ function ingestHook(body, pushSource = null) {
     transcriptCache.extract(data.transcript_path);
   }
 
-  const result = processEvent(hook_type, data, tool);
+  // The hook's frames also wait for bindRunSession: in server mode a member
+  // is sent only frames of sessions they own (broadcastAudience), and a new
+  // Codex/agy session has no workspace — no owner — until that stamp. They are
+  // dropped only when processEvent threw (nothing was committed).
+  held = [];
+  let result = null;
+  try {
+    result = processEvent(hook_type, data, tool);
+    if (result) {
+      if (pushSource) {
+        // Tagged in the same tick processEvent created it, so no local liveness
+        // sweep ever sees it as this machine's.
+        if (pushedNew) {
+          stmts.setSessionSource.run(pushSource, data.session_id);
+          broadcast("session_updated", stmts.getSession.get(data.session_id));
+        }
+      } else {
+        touchLaneFromHook(hook_type, data, tool, body.run_id);
+        bindRunSession(body.run_id, data.session_id, hook_type, data.source, tool);
+      }
+    }
+  } finally {
+    const frames = held;
+    held = null;
+    if (result) for (const [type, payload] of frames) sendNow(type, payload);
+  }
   if (!result) {
     return {
       status: 400,
       body: { error: { code: "MISSING_SESSION", message: "session_id is required in data" } },
     };
   }
-
-  if (pushSource) {
-    // Tagged in the same tick processEvent created it, so no local liveness
-    // sweep ever sees it as this machine's.
-    if (pushedNew) {
-      stmts.setSessionSource.run(pushSource, data.session_id);
-      broadcast("session_updated", stmts.getSession.get(data.session_id));
-    }
-  } else {
-    // ponytail: full lane scan per hook — fine at dozens of lanes; index by cwd prefix if lane counts grow.
-    touchLaneFromHook(hook_type, data, tool);
-    bindRunSession(body.run_id, data.session_id, hook_type, data.source, tool);
-  }
   return { status: 200, body: { ok: true, event: result }, result, hook_type, data, tool };
 }
 
@@ -1880,7 +1942,8 @@ function watchdogCheck() {
     //     sessions never flicker out on a transient mismatch. A false
     //     completion self-heals anyway: the next hook event reactivates the
     //     session via the existing needsReactivation path.
-    livenessReap();
+    // The probe's `ps` runs asynchronously here (not on the event loop).
+    return livenessReap({ asyncProbe: true });
   } catch (err) {
     // Watchdog is best-effort — log but never crash the server
     console.warn("[WATCHDOG] Error during check:", err?.message || err);
@@ -1901,7 +1964,7 @@ function watchdogCheck() {
  * existing reactivation path. Watchdog ticks keep the gate, where it still
  * guards long-running steady-state work against transient probe misses.
  */
-function livenessReap({ ignoreIdleGate = false } = {}) {
+function livenessReap({ ignoreIdleGate = false, asyncProbe = false, probe = null } = {}) {
   const fs = require("fs");
   const path = require("path");
 
@@ -1924,7 +1987,14 @@ function livenessReap({ ignoreIdleGate = false } = {}) {
     .all();
   if (activeSessions.length === 0) return; // nothing to check — skip the ps/lsof cost
 
-  const probe = liveness.probeLiveCwds();
+  if (asyncProbe) {
+    // Re-enters with the answer; the session list is read again then, fresh.
+    return liveness
+      .probeLiveCwdsAsync()
+      .then((p) => livenessReap({ ignoreIdleGate, probe: p }))
+      .catch((err) => console.warn("[WATCHDOG] Liveness reap failed:", err?.message || err));
+  }
+  if (!probe) probe = liveness.probeLiveCwds();
   if (!probe.available) return;
   const now = Date.now();
   for (const sess of activeSessions) {

server/routes/lanes.js

@@ -28,6 +28,7 @@ const laneFeatures = require("../lib/lane-features");
 const proofLib = require("../lib/proof");
 const { broadcast } = require("../websocket");
 const runs = require("../lib/pty-run");
+const tmux = require("../lib/tmux");
 const { resumeProxy } = require("../lib/proxy-apply");
 const { sameOriginGuard } = require("./run");
 const { writeLaneFile } = require("../lib/lane-file-write");
@@ -115,10 +116,14 @@ function lastEventAge(lane) {
  * it, exactly like the old push-based handler did, just pulled instead of
  * pushed.
  */
-function healRunId(lane) {
+function healRunId(lane, live = null) {
   if (!lane.run_id) return lane;
-  const run = runs.getRun(lane.run_id);
-  if (run && run.status === "running") return lane;
+  if (live) {
+    if (live.has(lane.run_id)) return lane;
+  } else {
+    const run = runs.getRun(lane.run_id);
+    if (run && run.status === "running") return lane;
+  }
   lanesLib.updateLane(lane.id, { run_id: null, status: "idle" });
   broadcastLane(lane.id);
   return lanesLib.getLane(lane.id);
@@ -128,10 +133,11 @@ function healRunId(lane) {
  * Heal every lane at once. A claude pane that exits fires a hook, which
  * broadcasts the lane and so re-reads it; a codex or antigravity pane fires
  * nothing, so without this sweep its lane read "running" until something
- * else happened to fetch it.
+ * else happened to fetch it. `live` (a `tmux.liveSessions()` set) answers
+ * every lane from one tmux call instead of a has-session per lane.
  */
-function healAllRunIds() {
-  for (const lane of lanesLib.listLanes()) if (lane.run_id) healRunId(lane);
+function healAllRunIds(live = null) {
+  for (const lane of lanesLib.listLanes()) if (lane.run_id) healRunId(lane, live);
 }
 
 /**
@@ -171,8 +177,8 @@ function broadcastForeignLaneChanges() {
   return out;
 }
 
-function payload(lane) {
-  const healed = healRunId(lane);
+function payload(lane, live = null) {
+  const healed = healRunId(lane, live);
   return lanesLib.lanePayload(healed, lastEventAge(healed));
 }
 
@@ -195,7 +201,11 @@ function broadcastLane(id) {
 }
 
 router.get("/", (_req, res) => {
-  const lanes = lanesLib.listLanes().map(payload);
+  const rows = lanesLib.listLanes();
+  // ONE list-sessions answers every lane's run, instead of a synchronous
+  // has-session per lane (null when tmux could not be asked: per-lane then).
+  const live = rows.some((l) => l.run_id) ? tmux.liveSessions() : null;
+  const lanes = rows.map((l) => payload(l, live));
   res.json({
     lanes,
     counts: {

server/routes/workflows.js

@@ -37,7 +37,14 @@ router.get("/", async (req, res) => {
     }
   }
   try {
-    res.json(await runRead("workflows", { status: statusFilter, tool, range, sections }));
+    res.json(
+      await runRead(
+        "workflows",
+        { status: statusFilter, tool, range, sections },
+        // Every open Workflows page shares one run (~1.5 s on a large DB).
+        { maxAgeMs: 30_000 }
+      )
+    );
   } catch (err) {
     if (err.code === "EREADTIMEOUT") {
       return res.status(503).json({ error: { code: err.code, message: err.message } });
@@ -73,7 +80,7 @@ router.get("/skill-usage", (req, res) => {
                 COUNT(*) AS count,
                 COUNT(DISTINCT session_id) AS sessions
          FROM events
-         WHERE tool_name = 'Skill' AND event_type = 'PreToolUse'
+         WHERE tool_name = 'Skill' AND +event_type = 'PreToolUse'
            AND json_extract(data, '$.tool_input.skill') IS NOT NULL${sf.clause}
          GROUP BY name ORDER BY count DESC LIMIT 30`
       )
@@ -83,11 +90,14 @@ router.get("/skill-usage", (req, res) => {
         return { ...r, plugin: i > 0 ? r.name.slice(0, i) : null };
       });
 
+    // A range on tool_name ("`" sorts right after "_") instead of LIKE, and
+    // `+event_type` to keep the planner off idx_events_type: both make the
+    // query use idx_events_tool (784 ms → a few ms on 253K events).
     const mcpCalls = db
       .prepare(
         `SELECT tool_name, COUNT(*) AS count FROM events
-         WHERE tool_name LIKE 'mcp__plugin\\_%' ESCAPE '\\'
-           AND event_type = 'PreToolUse'${sf.clause}
+         WHERE tool_name >= 'mcp__plugin_' AND tool_name < 'mcp__plugin\`'
+           AND +event_type = 'PreToolUse'${sf.clause}
          GROUP BY tool_name`
       )
       .all(...sf.params);