Smartgift-AI/AgentHub
14
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases 86 Wiki Activity

fix(server-s3): transcript snapshot copies asynchronously (F42 follow-up)

Browse Source 
copyIfNewer read/wrote a member file of up to 256 MiB with readSync/writeSync,
blocking the event loop for every user. It now copies with fs.promises 1 MiB
at a time from the runAs.openSafeAsync handle (owner/type check kept), keeps
the 256 MiB member cap and zero-run holes, writes a temp file and renames it
into place, and joins a copy already in flight for the same destination.
A failed copy is logged and never rejects; snapshotTranscript is async and
all three import callers await it.
This commit is contained in:
nntrivi2001
2026-10-09 15:17:55 +07:00
parent 84e2a9f0ac
commit 3ab750c765
6 changed files with 130 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/__tests__/server-s3-review3.test.js
+1 -1
View File
@@ -229,7 +229,7 @@ describe("a member's planted link is never followed with the dashboard's rights"
fs.mkdirSync(path.join(proj, sid, "subagents"), { recursive: true });
fs.writeFileSync(path.join(proj, `${sid}.jsonl`), '{"own":1}\n');
fs.writeFileSync(path.join(proj, sid, "subagents", "agent-own.jsonl"), '{"own":2}\n');
snapshotTranscript(path.join(proj, `${sid}.jsonl`), sid);
await snapshotTranscript(path.join(proj, `${sid}.jsonl`), sid);
const snap = getTranscriptSnapshotDir();
assert.equal(fs.readFileSync(path.join(snap, `${sid}.jsonl`), "utf8"), '{"own":1}\n');
assert.ok(fs.existsSync(path.join(snap, sid, "subagents", "agent-own.jsonl")));
server/__tests__/server-s3-review5.test.js
+75 -4
View File
@@ -7,7 +7,8 @@
* database); a stopped or flooding member tmux server and a hung sudo never
* block the dashboard nor blind it to other members' panes; a one-line 512 MB
* file, a huge member file and a sparse transcript neither crash the server
* nor fill the host's disk; an async route that throws answers 500; a
* nor fill the host's disk, and the snapshot copy yields to the event loop
* and runs once per file; an async route that throws answers 500; a
* non-admin owns no host-folder workspace and a pane's cwd is its
* workspace's account's; a pane token works only from its pane's uid; an
* analyst's GET path stays in their own folder; a drop goes to the pane's
@@ -370,13 +371,83 @@ describe("a member's huge or sparse files", () => {
fs.rmSync(t);
});
it("the transcript snapshot never fills the host's disk from a sparse file", () => {
it("the transcript snapshot copy yields to the event loop between chunks", async () => {
const { snapshotTranscript } = require("../../scripts/import-history");
const { getTranscriptSnapshotDir } = require("../lib/claude-home");
const proj = path.join(MIA, ".claude", "projects", "-p");
const sid = "11111111-2222-4333-8444-777777777777";
const src = path.join(proj, `${sid}.jsonl`);
fs.mkdirSync(proj, { recursive: true });
fs.writeFileSync(src, Buffer.alloc(64 * 1024 * 1024, 0x61));
let ticks = 0;
let maxGap = 0;
let last = performance.now();
let done = false;
const tick = () => {
const now = performance.now();
maxGap = Math.max(maxGap, now - last);
last = now;
ticks += 1;
if (!done) setImmediate(tick);
};
setImmediate(tick);
try {
await snapshotTranscript(src, sid);
} finally {
done = true;
}
assert.ok(ticks >= 8, `only ${ticks} event-loop ticks ran during the copy`);
assert.ok(maxGap < 50, `the event loop was blocked for ${maxGap.toFixed(1)} ms`);
const copy = path.join(getTranscriptSnapshotDir(), `${sid}.jsonl`);
assert.ok(fs.readFileSync(copy).equals(fs.readFileSync(src)));
fs.rmSync(src);
fs.rmSync(copy);
});
it("two overlapping snapshots copy a file once, and a failed copy only logs", async () => {
const { snapshotTranscript } = require("../../scripts/import-history");
const { getTranscriptSnapshotDir } = require("../lib/claude-home");
const proj = path.join(MIA, ".claude", "projects", "-p");
const sid = "11111111-2222-4333-8444-888888888888";
const src = path.join(proj, `${sid}.jsonl`);
fs.mkdirSync(path.join(proj, sid, "subagents"), { recursive: true });
fs.writeFileSync(src, '{"a":1}\n'.repeat(1 << 18));
fs.writeFileSync(path.join(proj, sid, "subagents", "agent-x.jsonl"), '{"b":1}\n');
// The subagent's snapshot folder cannot be made: its parent is a file.
fs.mkdirSync(getTranscriptSnapshotDir(), { recursive: true });
fs.writeFileSync(path.join(getTranscriptSnapshotDir(), sid), "");
const realRename = fs.promises.rename;
const realWarn = console.warn;
let renames = 0;
const warned = [];
fs.promises.rename = (...a) => {
renames += 1;
return realRename(...a);
};
console.warn = (...a) => warned.push(a.join(" "));
try {
await Promise.all([snapshotTranscript(src, sid), snapshotTranscript(src, sid)]);
} finally {
fs.promises.rename = realRename;
console.warn = realWarn;
}
assert.equal(renames, 1);
const copy = path.join(getTranscriptSnapshotDir(), `${sid}.jsonl`);
assert.ok(fs.readFileSync(copy).equals(fs.readFileSync(src)));
assert.ok(warned.some((w) => w.includes("agent-x.jsonl")), JSON.stringify(warned));
assert.deepEqual(
fs.readdirSync(getTranscriptSnapshotDir()).filter((f) => f.endsWith(".tmp")),
[]
);
});
it("the transcript snapshot never fills the host's disk from a sparse file", async () => {
const { snapshotTranscript } = require("../../scripts/import-history");
const { getTranscriptSnapshotDir } = require("../lib/claude-home");
const proj = path.join(MIA, ".claude", "projects", "-p");
const huge = "11111111-2222-4333-8444-555555555555";
sparse(path.join(proj, `${huge}.jsonl`), 300 * 1024 * 1024);
snapshotTranscript(path.join(proj, `${huge}.jsonl`), huge);
await snapshotTranscript(path.join(proj, `${huge}.jsonl`), huge);
assert.equal(fs.existsSync(path.join(getTranscriptSnapshotDir(), `${huge}.jsonl`)), false);
// Under the cap: copied, and its hole stays a hole.
const holey = "11111111-2222-4333-8444-666666666666";
@@ -384,7 +455,7 @@ describe("a member's huge or sparse files", () => {
head: '{"a":1}\n',
tail: '{"z":1}\n',
});
snapshotTranscript(path.join(proj, `${holey}.jsonl`), holey);
await snapshotTranscript(path.join(proj, `${holey}.jsonl`), holey);
const copy = path.join(getTranscriptSnapshotDir(), `${holey}.jsonl`);
const st = fs.statSync(copy);
assert.equal(st.size, 32 * 1024 * 1024);
server/__tests__/workflow-transcript-resolution.test.js
+4 -4
View File
@@ -235,7 +235,7 @@ describe("snapshotTranscript (durable nested-layout fallback)", () => {
]);
});
it("preserves the nested layout into the snapshot so the read route resolves it", () => {
it("preserves the nested layout into the snapshot so the read route resolves it", async () => {
// Build an independent source session, snapshot it, then resolve from the
// snapshot dir exactly as the route's third fallback does.
const srcMain = path.join(ENC_DIR, `${SESSION_SNAP}.jsonl`);
@@ -250,15 +250,15 @@ describe("snapshotTranscript (durable nested-layout fallback)", () => {
writeFile(srcMain, jsonl([{ type: "summary", summary: "snap" }]));
writeFile(srcNested, transcriptJsonl(PROMPT_TEXT, LONG_RESULT));
snapshotTranscript(srcMain, SESSION_SNAP);
await snapshotTranscript(srcMain, SESSION_SNAP);
const snapHit = getSnapshotSubagentTranscriptPath(SESSION_SNAP, WF_ID, RUN1);
assert.ok(snapHit, "snapshot resolver should find the preserved nested transcript");
assert.equal(fs.readFileSync(snapHit, "utf8"), fs.readFileSync(srcNested, "utf8"));
});
it("does not throw for a session with no transcripts", () => {
assert.doesNotThrow(() => snapshotTranscript(path.join(ENC_DIR, "nope.jsonl"), "nope"));
it("does not throw for a session with no transcripts", async () => {
await assert.doesNotReject(snapshotTranscript(path.join(ENC_DIR, "nope.jsonl"), "nope"));
});
});