perf: v2.5.10 — ready for 10+ concurrent users, leaner locally
Measured on a copy of a real 1.2 GB / 253K-event database, 10 simulated browsers + 10 hooks/s for 90 s: event-loop delay p99 1239 ms → 32 ms (max 2.0 s → 94 ms), 266 failed requests → 0, 74 → 111 requests/s served, server RSS 633 → 346 MB. Entry bundle 975 KB raw → 394 KB (106 KB brotli). Server - /api/analytics: the events-table scans run in the read worker (scoped-stats eventAggregates, read-pool job analyticsEvents); runRead takes maxAgeMs — analytics 10 s, /api/workflows 30 s — keyed by the args - /api/workflows/skill-usage: +event_type and a tool_name range instead of LIKE, so it uses idx_events_tool (784 ms → a few ms) - idx_events_session_type_tool (session_id, event_type, tool_name, created_at) replaces idx_events_session_type: covering reads for session stats, the hook dedupes and the import resume point (1.2 s → 20 ms); 64 MB cache_size and 256 MB mmap_size on both connections - WAL: the server sets wal_autocheckpoint = 0 and checkpoints PASSIVE every DASHBOARD_WAL_CHECKPOINT_MS (5 s) from the read worker's own writable connection; a failed checkpoint restores the automatic one (hook COMMIT p95 109 ms → 15 ms at 100 hooks/s) - tmux: one list-sessions answers a whole GET /api/lanes and the heal beat (liveSessions), shared for 1 s and cleared by this process's new/kill — GET /api/run's sync spawns had blocked 60 s of a 90 s run; the heal beat skips tmux when nothing runs; attach options chained into 2 spawns - hooks: TurnDuration/APIError deduped against one read per hook instead of a query per transcript entry (23 ms/hook on a 763-turn session); one lane scan per hook; hourly upserts skip rows that did not grow; frames are sent after the transaction commits (and after the pane binding); new_event carries its row id; a split pane's (slot 1–9) hooks no longer rebind the lane's session_id - watchdog liveness probe runs ps asynchronously; transcript newline scan via indexOf; startup subagent backfill fingerprints the files it already checked - pty attach: backpressure (pause past 1 MB unsent, resume under 256 KB) - static: hashed assets served from their pre-compressed .br/.gz sibling Client - hidden tabs neither refetch on WS events nor poll (useLiveRefresh, panes, lane cards, git header, changes panel, settings); no per-tab hourly /updates/check (the server already checks) - refetch storms throttled (workspace tree/session list 3 s, /api/run only on a run-key change, notification pane lookup only when shown); WS reconnect jitter; terminal retry backoff 1.5 → 30 s, stops on 1008 - heavier throttles for aggregates (15 s) and detail views; Monitor keepIfSame; GET in-flight dedupe in request() - lazy Dashboard/History, xterm (with the first terminal), lane modals, Sagi (only while enabled); only the active locale loads; Workspace chunk prefetched on a /workspace deep link - Sagi: no work while disabled, pupil via CSS variables, memo avatar - build: .br/.gz siblings, sourcemap hidden; service worker no longer caches /assets/ (cache v3 drops the old one) Tests: scripts/test-server.js drops the GIT_DIR-family variables a hook exports, so a suite run from a linked worktree's pre-commit can no longer commit into the real repository or set core.bare. Docs: ARCHITECTURE, DATABASE, API, DEPLOYMENT, client README, docs/plans/perf/README.md (audit, checklist, results).
This commit is contained in:
+110
-40
@@ -7,7 +7,15 @@ const { Router } = require("express");
|
||||
const { v4: uuidv4 } = require("uuid");
|
||||
const dbModule = require("../db");
|
||||
const { stmts, db } = dbModule;
|
||||
const { broadcast } = require("../websocket");
|
||||
const { broadcast: sendNow } = require("../websocket");
|
||||
|
||||
// Frames emitted while a hook's transaction is open wait in `held` for its
|
||||
// commit (see processEvent); everywhere else they go out at once.
|
||||
let held = null;
|
||||
function broadcast(type, data) {
|
||||
if (held) held.push([type, data]);
|
||||
else sendNow(type, data);
|
||||
}
|
||||
const TranscriptCache = require("../lib/transcript-cache");
|
||||
const { writeHourly } = require("../lib/hourly-usage");
|
||||
const { scanAndImportSubagents } = require("../../scripts/import-history");
|
||||
@@ -173,13 +181,16 @@ const LANE_DONE_HOOKS = new Set(["Stop", "SessionEnd"]);
|
||||
* running/idle, so without this an adopted lane read "idle" for the entire
|
||||
* time Claude was working in it.
|
||||
*/
|
||||
function touchLaneFromHook(hookType, data, tool = "claude") {
|
||||
function touchLaneFromHook(hookType, data, tool = "claude", runId = null) {
|
||||
try {
|
||||
if (!data || !data.cwd) return;
|
||||
// One read of the lane table for every path this hook resolves.
|
||||
// ponytail: a scan per hook — fine at dozens of lanes; index by cwd prefix if lane counts grow.
|
||||
const allLanes = lanesLib.listLanes();
|
||||
// Pin to the lane of the CLI that sent the hook — a Claude hook never
|
||||
// touches the codex lane sharing its directory, nor a Codex hook the
|
||||
// claude one.
|
||||
const lane = lanesLib.resolveLaneByCwd(data.cwd, tool);
|
||||
const lane = lanesLib.resolveLaneByCwd(data.cwd, tool, allLanes);
|
||||
if (!lane) return;
|
||||
|
||||
// Infer the lane's stage from this one tool event, best-effort. A real
|
||||
@@ -203,7 +214,7 @@ function touchLaneFromHook(hookType, data, tool = "claude") {
|
||||
// Credit the lane whose files/directory the tool actually touched, which
|
||||
// is not always the session's lane - see resolveLaneForWork. Bookkeeping
|
||||
// below still uses `lane`, the session's own.
|
||||
const workLane = lanesLib.resolveLaneForWork(data.cwd, data.tool_input) || lane;
|
||||
const workLane = lanesLib.resolveLaneForWork(data.cwd, data.tool_input, allLanes) || lane;
|
||||
const detection = detect(getPipeline(workLane.pipeline), {
|
||||
tool_name: data.tool_name,
|
||||
tool_input: data.tool_input,
|
||||
@@ -222,7 +233,14 @@ function touchLaneFromHook(hookType, data, tool = "claude") {
|
||||
// resuming, so it neither clears needs_action nor moves the lane.
|
||||
const childEvent = tool === "codex" && data._codexChild === true;
|
||||
const idleNudge = hookType === "Notification" && IDLE_NUDGE_RE.test(data.message || "");
|
||||
if (data.session_id && lane.session_id !== data.session_id) patch.session_id = data.session_id;
|
||||
// The lane's session follows its slot-0 pane (or a session outside the
|
||||
// dashboard), never a split pane's: only slot 0 is the lane's run, and two
|
||||
// panes of one workspace would otherwise flip session_id — and broadcast
|
||||
// the lane to every browser — on every hook.
|
||||
const splitPane = typeof runId === "string" && /^agenthub-lane-\d+-[1-9]$/.test(runId);
|
||||
if (data.session_id && lane.session_id !== data.session_id && !splitPane) {
|
||||
patch.session_id = data.session_id;
|
||||
}
|
||||
if (hookType === "Notification" && !idleNudge) {
|
||||
patch.needs_action = data.message || "needs you";
|
||||
patch.needs_action_session = data.session_id || null;
|
||||
@@ -260,6 +278,14 @@ function touchLaneFromHook(hookType, data, tool = "claude") {
|
||||
}
|
||||
|
||||
const setSessionTool = db.prepare("UPDATE sessions SET tool = ? WHERE id = ?");
|
||||
// What processEvent dedupes the transcript's API errors and turn durations
|
||||
// against — one read per hook, not one per transcript entry.
|
||||
const apiErrorSummariesStmt = db.prepare(
|
||||
"SELECT summary FROM events WHERE session_id = ? AND event_type = 'APIError'"
|
||||
);
|
||||
const turnDurationStampsStmt = db.prepare(
|
||||
"SELECT created_at FROM events WHERE session_id = ? AND event_type = 'TurnDuration'"
|
||||
);
|
||||
|
||||
/** The session's `<id>-main` agent, created and broadcast. */
|
||||
function createMainAgent(session) {
|
||||
@@ -448,7 +474,7 @@ function applyFirstUserDescriptor(sessionId, result) {
|
||||
if (refreshedAgent) broadcast("agent_updated", refreshedAgent);
|
||||
}
|
||||
|
||||
const processEvent = db.transaction((hookType, data, tool = "claude") => {
|
||||
const processEventTx = db.transaction((hookType, data, tool = "claude") => {
|
||||
const sessionId = data.session_id;
|
||||
if (!sessionId) return null;
|
||||
|
||||
@@ -1017,16 +1043,13 @@ const processEvent = db.transaction((hookType, data, tool = "claude") => {
|
||||
// Register API errors from transcript (quota limits, rate limits, overloaded, etc.)
|
||||
if (result.errors) {
|
||||
let newErrorRecorded = false;
|
||||
// Deduplicate against the errors already recorded, read ONCE per hook:
|
||||
// `result.errors` is the transcript's whole history, not just new ones.
|
||||
const recorded = new Set(apiErrorSummariesStmt.all(sessionId).map((r) => r.summary));
|
||||
for (const apiErr of result.errors) {
|
||||
// Deduplicate: check if we already recorded this error (same type+message+timestamp)
|
||||
const errKey = `${apiErr.type}:${apiErr.timestamp || ""}`;
|
||||
const existing = db
|
||||
.prepare(
|
||||
`SELECT 1 FROM events WHERE session_id = ? AND event_type = 'APIError'
|
||||
AND summary = ? LIMIT 1`
|
||||
)
|
||||
.get(sessionId, `${apiErr.type}: ${apiErr.message}`);
|
||||
if (existing) continue;
|
||||
const errSummary = `${apiErr.type}: ${apiErr.message}`;
|
||||
if (recorded.has(errSummary)) continue;
|
||||
recorded.add(errSummary);
|
||||
|
||||
stmts.insertEvent.run(
|
||||
sessionId,
|
||||
@@ -1096,15 +1119,15 @@ const processEvent = db.transaction((hookType, data, tool = "claude") => {
|
||||
|
||||
// Register turn duration events from transcript
|
||||
if (result.turnDurations) {
|
||||
// `turnDurations` is every turn in the transcript (up to 1000), on
|
||||
// every hook: dedupe against ONE read of the recorded timestamps (a
|
||||
// covering-index scan) instead of a query per turn — 23 ms per hook on
|
||||
// a 763-turn session before.
|
||||
const recorded = new Set(turnDurationStampsStmt.all(sessionId).map((r) => r.created_at));
|
||||
for (const td of result.turnDurations) {
|
||||
const tdTs = td.timestamp || new Date().toISOString();
|
||||
// Deduplicate by checking if we already have this turn duration event
|
||||
const existing = db
|
||||
.prepare(
|
||||
"SELECT 1 FROM events WHERE session_id = ? AND event_type = 'TurnDuration' AND created_at = ? LIMIT 1"
|
||||
)
|
||||
.get(sessionId, tdTs);
|
||||
if (existing) continue;
|
||||
if (recorded.has(tdTs)) continue;
|
||||
recorded.add(tdTs);
|
||||
|
||||
const tdSummary = `Turn completed in ${(td.durationMs / 1000).toFixed(1)}s`;
|
||||
// Stamped with the TRANSCRIPT's timestamp, not NOW — that is what the
|
||||
@@ -1168,7 +1191,7 @@ const processEvent = db.transaction((hookType, data, tool = "claude") => {
|
||||
// Bump session updated_at on every event
|
||||
stmts.touchSession.run(sessionId);
|
||||
|
||||
stmts.insertEvent.run(
|
||||
const inserted = stmts.insertEvent.run(
|
||||
sessionId,
|
||||
agentId,
|
||||
eventType,
|
||||
@@ -1179,6 +1202,8 @@ const processEvent = db.transaction((hookType, data, tool = "claude") => {
|
||||
);
|
||||
|
||||
const event = {
|
||||
// The row id (additive): lets a client drop a frame it already holds.
|
||||
id: Number(inserted.lastInsertRowid),
|
||||
session_id: sessionId,
|
||||
agent_id: agentId,
|
||||
event_type: eventType,
|
||||
@@ -1190,6 +1215,31 @@ const processEvent = db.transaction((hookType, data, tool = "claude") => {
|
||||
return event;
|
||||
});
|
||||
|
||||
/**
|
||||
* One hook's writes in one transaction, its WebSocket frames sent only once
|
||||
* that transaction COMMITTED — in order, in the same tick. A frame sent from
|
||||
* inside it described a row a later throw could still roll back.
|
||||
*/
|
||||
function processEvent(hookType, data, tool) {
|
||||
return holdFrames(() => processEventTx(hookType, data, tool));
|
||||
}
|
||||
|
||||
/** Run `fn` with frames held, then send them if it returned (nested: the
|
||||
* outermost call sends). A throw drops them. */
|
||||
function holdFrames(fn) {
|
||||
if (held) return fn();
|
||||
held = [];
|
||||
let frames = null;
|
||||
try {
|
||||
const out = fn();
|
||||
frames = held;
|
||||
return out;
|
||||
} finally {
|
||||
held = null;
|
||||
if (frames) for (const [type, payload] of frames) sendNow(type, payload);
|
||||
}
|
||||
}
|
||||
|
||||
// A Codex sub-agent thread's own lifecycle is not the user's session's: its
|
||||
// Stop is not the end of the user's turn, nor its SessionStart a new session.
|
||||
const CODEX_CHILD_DROPPED = new Set([
|
||||
@@ -1414,26 +1464,38 @@ function ingestHook(body, pushSource = null) {
|
||||
transcriptCache.extract(data.transcript_path);
|
||||
}
|
||||
|
||||
const result = processEvent(hook_type, data, tool);
|
||||
// The hook's frames also wait for bindRunSession: in server mode a member
|
||||
// is sent only frames of sessions they own (broadcastAudience), and a new
|
||||
// Codex/agy session has no workspace — no owner — until that stamp. They are
|
||||
// dropped only when processEvent threw (nothing was committed).
|
||||
held = [];
|
||||
let result = null;
|
||||
try {
|
||||
result = processEvent(hook_type, data, tool);
|
||||
if (result) {
|
||||
if (pushSource) {
|
||||
// Tagged in the same tick processEvent created it, so no local liveness
|
||||
// sweep ever sees it as this machine's.
|
||||
if (pushedNew) {
|
||||
stmts.setSessionSource.run(pushSource, data.session_id);
|
||||
broadcast("session_updated", stmts.getSession.get(data.session_id));
|
||||
}
|
||||
} else {
|
||||
touchLaneFromHook(hook_type, data, tool, body.run_id);
|
||||
bindRunSession(body.run_id, data.session_id, hook_type, data.source, tool);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
const frames = held;
|
||||
held = null;
|
||||
if (result) for (const [type, payload] of frames) sendNow(type, payload);
|
||||
}
|
||||
if (!result) {
|
||||
return {
|
||||
status: 400,
|
||||
body: { error: { code: "MISSING_SESSION", message: "session_id is required in data" } },
|
||||
};
|
||||
}
|
||||
|
||||
if (pushSource) {
|
||||
// Tagged in the same tick processEvent created it, so no local liveness
|
||||
// sweep ever sees it as this machine's.
|
||||
if (pushedNew) {
|
||||
stmts.setSessionSource.run(pushSource, data.session_id);
|
||||
broadcast("session_updated", stmts.getSession.get(data.session_id));
|
||||
}
|
||||
} else {
|
||||
// ponytail: full lane scan per hook — fine at dozens of lanes; index by cwd prefix if lane counts grow.
|
||||
touchLaneFromHook(hook_type, data, tool);
|
||||
bindRunSession(body.run_id, data.session_id, hook_type, data.source, tool);
|
||||
}
|
||||
return { status: 200, body: { ok: true, event: result }, result, hook_type, data, tool };
|
||||
}
|
||||
|
||||
@@ -1880,7 +1942,8 @@ function watchdogCheck() {
|
||||
// sessions never flicker out on a transient mismatch. A false
|
||||
// completion self-heals anyway: the next hook event reactivates the
|
||||
// session via the existing needsReactivation path.
|
||||
livenessReap();
|
||||
// The probe's `ps` runs asynchronously here (not on the event loop).
|
||||
return livenessReap({ asyncProbe: true });
|
||||
} catch (err) {
|
||||
// Watchdog is best-effort — log but never crash the server
|
||||
console.warn("[WATCHDOG] Error during check:", err?.message || err);
|
||||
@@ -1901,7 +1964,7 @@ function watchdogCheck() {
|
||||
* existing reactivation path. Watchdog ticks keep the gate, where it still
|
||||
* guards long-running steady-state work against transient probe misses.
|
||||
*/
|
||||
function livenessReap({ ignoreIdleGate = false } = {}) {
|
||||
function livenessReap({ ignoreIdleGate = false, asyncProbe = false, probe = null } = {}) {
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
@@ -1924,7 +1987,14 @@ function livenessReap({ ignoreIdleGate = false } = {}) {
|
||||
.all();
|
||||
if (activeSessions.length === 0) return; // nothing to check — skip the ps/lsof cost
|
||||
|
||||
const probe = liveness.probeLiveCwds();
|
||||
if (asyncProbe) {
|
||||
// Re-enters with the answer; the session list is read again then, fresh.
|
||||
return liveness
|
||||
.probeLiveCwdsAsync()
|
||||
.then((p) => livenessReap({ ignoreIdleGate, probe: p }))
|
||||
.catch((err) => console.warn("[WATCHDOG] Liveness reap failed:", err?.message || err));
|
||||
}
|
||||
if (!probe) probe = liveness.probeLiveCwds();
|
||||
if (!probe.available) return;
|
||||
const now = Date.now();
|
||||
for (const sess of activeSessions) {
|
||||
|
||||
Reference in New Issue
Block a user