feat(db): trim stored hook payloads; price the session list like a session

Analytics parity, phase 1 (data hygiene), from the analytics-parity session.

events.data is now stored through the SQL function trim_event_data(?)
(server/lib/event-payload.js, registered in server/db.js) on every event
INSERT and in the duplicate guard: Edit/MultiEdit/NotebookEdit/Write drop
the whole-file mirrors (originalFile, oldString/newString, Write content),
Read drops file.content/base64, and any string in tool_input/tool_response
over 8 KB keeps a 6 KB head + 2 KB tail. structuredPatch and every field
outside those two stay whole; _trimmed names what was dropped. The full
text stays in the transcript and its snapshot. Rows stored before this are
trimmed once in the background (sweepEventPayloads, 200 rows per write
transaction, user_version 4). No VACUUM, no row pruned.

The session list now prices every token column /api/pricing/cost/:id does
(speed, geo, tier, the 1h cache write, server-tool fees) through one
tokensForSessions helper, so a row's cost equals the session's own.
This commit is contained in:
2026-10-05 13:13:02 +07:00
parent 138e706dd1
commit 215cae7393
9 changed files with 368 additions and 46 deletions
+143
View File
@@ -0,0 +1,143 @@
/**
* @file event-payload.js
* @description Trims a hook payload before it is stored as `events.data`. A
* PostToolUse payload carries whole files — Edit's `originalFile`, Read's file
* `content`/`base64`, Write's `content` — and long command output, none of which
* any analysis reads: the server reads only `tool_input.skill`, the event views
* only preview. The full text stays in the session's transcript (and AgentHub's
* snapshot of it under `<data dir>/transcripts/`), which the Conversation tab
* reads. Kept: every field outside `tool_input`/`tool_response`, every number and
* flag, `structuredPatch` (the +/- line counts), and each long string's head and
* tail. Dropped fields are named in `_trimmed`, so a view can say so.
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
*/
// Strings inside tool_input / tool_response longer than this keep head + tail.
const STR_CAP = 8192;
const HEAD = 6000;
const TAIL = 2000;
// A payload this short cannot hold anything worth trimming, so it is not parsed.
const SKIP_UNDER = STR_CAP;
// Per tool, the tool_response fields that mirror a whole file or repeat
// tool_input. Other tools' fields (Grep's `content` is its matches) only get capped.
const EDIT_DROP = ["originalFile", "oldString", "newString"];
const DROP_RESPONSE = {
Edit: EDIT_DROP,
MultiEdit: EDIT_DROP,
NotebookEdit: EDIT_DROP,
Write: [...EDIT_DROP, "content"],
};
const DROP_RESPONSE_FILE = { Read: ["content", "base64"] };
function capString(s) {
if (s.length <= STR_CAP) return s;
return `${s.slice(0, HEAD)}\n…[${s.length - HEAD - TAIL} chars trimmed]…\n${s.slice(-TAIL)}`;
}
// Caps every string at any depth; `structuredPatch` is left whole.
function capDeep(v, key) {
if (typeof v === "string") return capString(v);
if (key === "structuredPatch" || v === null || typeof v !== "object") return v;
if (Array.isArray(v)) return v.map((x) => capDeep(x));
const out = {};
for (const [k, x] of Object.entries(v)) out[k] = capDeep(x, k);
return out;
}
/**
* @param {object} data A parsed hook payload.
* @returns {object} A trimmed copy (the input is not mutated).
*/
function trimHookPayload(data) {
if (!data || typeof data !== "object" || Array.isArray(data)) return data;
const out = { ...data };
const dropped = new Set(Array.isArray(data._trimmed) ? data._trimmed : []);
const res = out.tool_response;
if (res && typeof res === "object" && !Array.isArray(res)) {
const r = { ...res };
for (const k of DROP_RESPONSE[data.tool_name] || []) {
if (typeof r[k] === "string" && r[k].length > 0) {
delete r[k];
dropped.add(`tool_response.${k}`);
}
}
const fileDrop = DROP_RESPONSE_FILE[data.tool_name];
if (fileDrop && r.file && typeof r.file === "object") {
const f = { ...r.file };
for (const k of fileDrop) {
if (typeof f[k] === "string" && f[k].length > 0) {
delete f[k];
dropped.add(`tool_response.file.${k}`);
}
}
r.file = f;
}
out.tool_response = r;
}
if ("tool_response" in out) out.tool_response = capDeep(out.tool_response);
if ("tool_input" in out) out.tool_input = capDeep(out.tool_input);
if (dropped.size > 0) out._trimmed = [...dropped];
return out;
}
/**
* The SQL-side entry point (`trim_event_data(?)`, registered in `server/db.js`):
* a JSON string in, a JSON string out. Short, null or unparseable input is
* returned unchanged, and trimming is idempotent.
* @param {string|null} json
* @returns {string|null}
*/
function trimEventData(json) {
if (typeof json !== "string" || json.length <= SKIP_UNDER) return json;
let parsed;
try {
parsed = JSON.parse(json);
} catch {
return json;
}
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return json;
return JSON.stringify(trimHookPayload(parsed));
}
/**
* Trims the rows stored before trimming existed, a chunk per write transaction
* so a hook never waits long for the lock. Idempotent: a crash just resumes from
* the start on the next run.
* @param {import("better-sqlite3").Database} db
* @param {{chunk?: number}} [opts]
* @returns {Promise<{rows: number, bytesSaved: number}>}
*/
function sweepEventPayloads(db, { chunk = 200 } = {}) {
const select = db.prepare(
"SELECT id, data FROM events WHERE id > ? AND length(data) > ? ORDER BY id LIMIT ?"
);
const update = db.prepare("UPDATE events SET data = ? WHERE id = ?");
const out = { rows: 0, bytesSaved: 0 };
const trimChunk = db.transaction((rows) => {
for (const r of rows) {
const t = trimEventData(r.data);
if (t === r.data) continue;
update.run(t, r.id);
out.rows++;
out.bytesSaved += r.data.length - t.length;
}
});
let lastId = 0;
return new Promise((resolve, reject) => {
const step = () => {
try {
const rows = select.all(lastId, SKIP_UNDER, chunk);
if (rows.length === 0) return resolve(out);
trimChunk(rows);
lastId = rows[rows.length - 1].id;
setImmediate(step);
} catch (err) {
reject(err);
}
};
step();
});
}
module.exports = { trimHookPayload, trimEventData, sweepEventPayloads, STR_CAP };