fix(drop): stream every dropped file to disk; no local size cap

- POST /api/run/drop-file pipes the body to its file as it arrives in every case — a member's (S3) through `write-as … replace` with the bytes on stdin — instead of buffering up to 20 MB in memory for a remote browser. - Remote limit (server mode, a LAN browser) 20 MB → 500 MB, as a Files upload. - A loopback browser in local mode has no cap any more, only the data dir's free space less a 1 GB reserve; ETOOBIG's max_bytes is what still fits, so the UI's "at most N MB" stays true. - drop-image keeps its 20 MB in-memory limit, now checked on its own. - fake-sudo gains a spawn for run-as.__setSpawnImpl.
2026-10-26 16:36:29 +07:00
parent a0cd808456
commit dce32b96b7
6 changed files with 98 additions and 73 deletions
@@ -1,32 +1,34 @@
 /**
 * @file fake-sudo.js
 * @description Test helper for the S3 suites: an `execImpl` for
- * `run-as.__setExecImpl` that plays `sudo -n -u <login> [VAR=v…] <bin> …`
- * without root — it records the argv (so a test can assert nothing secret
+ * `run-as.__setExecImpl` (and a `spawn` for `__setSpawnImpl`) that plays
+ * `sudo -n -u <login> [VAR=v…] <bin> …` without root — it records the argv (so a test can assert nothing secret
 * crossed sudo) and runs the real `scripts/server/write-as.sh` for
 * `/opt/agenthub/libexec/write-as`, as this account.
 * @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
 */

 const path = require("node:path");
-const { execFileSync } = require("node:child_process");
+const { execFileSync, spawn: spawnChild } = require("node:child_process");

 const WRITE_AS = path.join(__dirname, "..", "..", "..", "scripts", "server", "write-as.sh");

-/** `{ exec, calls }`: pass `exec` to `runAs.__setExecImpl`; `calls` = every sudo argv. */
+/** `{ exec, spawn, calls }`: pass `exec` to `runAs.__setExecImpl` and `spawn`
+ *  to `runAs.__setSpawnImpl`; `calls` = every sudo argv. */
 function fakeSudo() {
  const calls = [];
-  const exec = (cmd, argv, opts = {}) => {
+  // The command sudo would run, as [file, args].
+  const unwrap = (cmd, argv) => {
    calls.push([cmd, ...argv]);
-    if (cmd !== "sudo") return execFileSync(cmd, argv, opts);
+    if (cmd !== "sudo") return [cmd, argv];
    let i = argv.indexOf("-u") + 2; // past the login
    while (/^[A-Z_][A-Z0-9_]*=/.test(argv[i])) i++;
    const [bin, ...rest] = argv.slice(i);
-    if (bin === "/opt/agenthub/libexec/write-as")
-      return execFileSync("sh", [WRITE_AS, ...rest], opts);
-    return execFileSync(bin, rest, opts);
+    return bin === "/opt/agenthub/libexec/write-as" ? ["sh", [WRITE_AS, ...rest]] : [bin, rest];
  };
-  return { exec, calls };
+  const exec = (cmd, argv, opts = {}) => execFileSync(...unwrap(cmd, argv), opts);
+  const spawn = (cmd, argv, opts = {}) => spawnChild(...unwrap(cmd, argv), opts);
+  return { exec, spawn, calls };
 }

 module.exports = { fakeSudo };
@@ -334,12 +334,12 @@ describe("POST /api/run/drop-image", () => {
  });

  it("refuses a file declared over the limit before its body arrives", async () => {
-    // 4 GB announced, 1 KB sent: the answer must come without the rest.
+    // A petabyte announced, 1 KB sent: the answer must come without the rest.
    const { status, body } = await new Promise((resolve, reject) => {
      const url = new URL("/api/run/drop-file?name=big.mp4", BASE);
      const req = http.request(url, {
        method: "POST",
-        headers: { "Content-Type": "video/mp4", "Content-Length": 4 * 1024 ** 3 },
+        headers: { "Content-Type": "video/mp4", "Content-Length": 1024 ** 5 },
      });
      req.on("response", (res) => {
        let text = "";
@@ -354,8 +354,10 @@ describe("POST /api/run/drop-image", () => {
    });
    assert.equal(status, 413);
    assert.equal(body.error.code, "ETOOBIG");
-    // The UI shows the server's limit: a loopback browser's, here.
-    assert.equal(body.error.max_bytes, 2 * 1024 ** 3);
+    // A loopback browser has no cap, only the disk's free space (less a
+    // reserve) — and that is the limit the UI shows.
+    const st = fs.statfsSync(process.env.DASHBOARD_DATA_DIR);
+    assert.ok(body.error.max_bytes <= st.bavail * st.bsize, String(body.error.max_bytes));
  });

  it("streams a loopback drop over the remote limit to disk", async () => {
@@ -97,7 +97,9 @@ before(async () => {
  );
  linux.__resetConfCache();
  linux.__setRunImpl(() => "ok");
-  runAs.__setExecImpl(fakeSudo().exec);
+  const sudo = fakeSudo();
+  runAs.__setExecImpl(sudo.exec);
+  runAs.__setSpawnImpl(sudo.spawn);
  tmux.__setExecImpl(fakeTmux);
  server = await startServer(createApp(), 0);
  BASE = `http://127.0.0.1:${server.address().port}`;
@@ -708,6 +710,8 @@ describe("a dropped file", () => {
      byAdmin.body.path.startsWith(path.join(MIA, ".agenthub", "handoff", "drops") + path.sep),
      byAdmin.body.path
    );
+    // Streamed to the member's file through write-as, not held whole.
+    assert.equal(fs.readFileSync(byAdmin.body.path, "utf8"), "hello");
    // A host workspace's pane runs as the host: the host's own 0700 drops dir.
    const hostLane = addLane(path.join(TMP, "hostdir", "d"), "boss");
    const host = await drop("boss", hostLane);
@@ -25,6 +25,8 @@ const express = require("express");
 const { Router } = express;
 const fs = require("node:fs");
 const crypto = require("node:crypto");
+const { Transform } = require("node:stream");
+const { pipeline } = require("node:stream/promises");
 const path = require("node:path");
 const runs = require("../lib/pty-run");
 const tmux = require("../lib/tmux");
@@ -114,21 +116,35 @@ const DROP_TYPES = {
  "image/webp": "webp",
 };
 const DROP_MAX_BYTES = 20 * 1024 * 1024;
-// A browser on this machine (loopback, never server mode) crosses no network:
-// its drop is streamed to disk, so a video or a dump is no reason to refuse.
-// Chrome hands a dropped file no `file://` path; the client first asks for
-// the file manager's own (`/selected-files`, `/clipboard-files`) and only
-// uploads what that cannot place.
-const DROP_LOCAL_MAX_BYTES = 2 * 1024 ** 3;
+// drop-file streams every body to disk, never holding it in memory: from
+// another machine (server mode, a LAN browser) up to what a Files upload may
+// carry, so a video or a dump dropped on a pane is no reason to refuse.
+const DROP_REMOTE_MAX_BYTES = 500 * 1024 * 1024;
+// A browser on this machine (loopback, never server mode) crosses no network,
+// so its drop has no size cap — only what the disk can take: the copy must
+// leave DROP_LOCAL_RESERVE_BYTES free. Chrome hands a dropped file no
+// `file://` path; the client first asks for the file manager's own
+// (`/selected-files`, `/clipboard-files`) and only uploads what that cannot
+// place.
+const DROP_LOCAL_RESERVE_BYTES = 1024 ** 3;
 const DROP_TTL_MS = 24 * 60 * 60_000;
 const localDrop = (req) => !require("../lib/auth").isServerMode() && isLocalRequest(req);
-const dropLimit = (req) => (localDrop(req) ? DROP_LOCAL_MAX_BYTES : DROP_MAX_BYTES);
+/** The free space a local drop may fill (a local drop lands in the data dir). */
+function localRoom() {
+  try {
+    const st = fs.statfsSync(require("../lib/claude-home").getDataDir());
+    return Math.max(0, st.bavail * st.bsize - DROP_LOCAL_RESERVE_BYTES);
+  } catch {
+    return Infinity; // unknown: a full disk still fails the write, and leaves no file
+  }
+}
+const dropLimit = (req) => (localDrop(req) ? localRoom() : DROP_REMOTE_MAX_BYTES);
 function tooBig(res, max) {
  res.set("Connection", "close");
  res.status(413).json({
    error: {
      code: "ETOOBIG",
-      message: `a dropped file may be at most ${max / 1024 / 1024} MB`,
+      message: `a dropped file may be at most ${Math.floor(max / 1024 / 1024)} MB`,
      max_bytes: max,
    },
  });
@@ -136,11 +152,11 @@ function tooBig(res, max) {
 /** A body declared over the limit is refused before a byte of it is read, and
 *  the connection is closed: express.raw would read (and discard) all of it
 *  first, so a 4 GB drop crossed the network in full just to be told 413. */
-function refuseOversize(req, res, next) {
-  const max = dropLimit(req);
+const refuseOversize = (limit) => (req, res, next) => {
+  const max = limit(req);
  if (!(Number(req.headers["content-length"]) > max)) return next();
  tooBig(res, max);
-}
+};
 /** Where a drop goes: the handoff dir of the account the PANE runs as — the
 *  workspace `?laneId=` names (scopeGuard already refused one this user may
 *  not see), else the requester's own — written AS that member under S3, so
@@ -195,7 +211,8 @@ function saveDrop(req, res, name, body) {

 router.post(
  "/drop-image",
-  refuseOversize,
+  // An image is held in memory: 20 MB, wherever the browser is.
+  refuseOversize(() => DROP_MAX_BYTES),
  express.raw({ type: Object.keys(DROP_TYPES), limit: DROP_MAX_BYTES }),
  (req, res) => {
    const ext =
@@ -222,63 +239,63 @@ const dropName = (req) =>
    .replace(/^\.+/, "")
    .slice(-100);

-/** A local browser's drop is piped to its file as it arrives, never held in
- *  memory whole; a body that runs past the limit (or is empty) leaves no file. */
-function streamLocalDrop(req, res, next) {
-  if (!localDrop(req)) return next();
+/** A drop is piped to its file as it arrives, never held in memory whole —
+ *  a member's (S3) through `write-as` as them, its bytes on stdin; a body that
+ *  runs past the limit (or is empty, or is cut off) leaves no file. */
+async function streamDrop(req, res) {
  const safe = dropName(req);
  if (!safe) return res.status(400).json({ error: "?name= and a non-empty file body required" });
+  const runAs = require("../lib/run-as");
+  const max = dropLimit(req);
  let file;
-  let out;
+  let member;
  try {
-    const { dir, member } = prepareDropDir(req);
-    if (member) return next(); // a member's file is written as them, by saveDrop
-    file = path.join(dir, `${crypto.randomBytes(8).toString("hex")}-${safe}`);
-    out = fs.createWriteStream(file, { mode: 0o600, flags: "wx" });
+    const dropped = prepareDropDir(req);
+    member = dropped.member;
+    file = path.join(dropped.dir, `${crypto.randomBytes(8).toString("hex")}-${safe}`);
  } catch (err) {
    return res.status(500).json({ error: err.message });
  }
  let bytes = 0;
-  let failed = null;
-  const fail = (status, send) => {
-    if (failed) return;
-    failed = status;
-    req.unpipe(out);
-    out.destroy();
-    fs.rm(file, { force: true }, () => {});
-    if (!res.headersSent) send();
-  };
-  req.on("data", (c) => {
-    bytes += c.length;
-    if (bytes > DROP_LOCAL_MAX_BYTES) fail(413, () => tooBig(res, DROP_LOCAL_MAX_BYTES));
+  const count = new Transform({
+    transform(chunk, _enc, cb) {
+      bytes += chunk.length;
+      if (bytes > max) return cb(Object.assign(new Error("too big"), { code: "ETOOBIG" }));
+      cb(null, chunk);
+    },
  });
-  req.on("aborted", () => fail(400, () => {}));
-  out.on("error", (err) => fail(500, () => res.status(500).json({ error: err.message })));
-  out.on("finish", () => {
-    if (failed) return;
-    if (bytes === 0) {
-      return fail(400, () =>
-        res.status(400).json({ error: "?name= and a non-empty file body required" })
-      );
+  try {
+    if (member) {
+      await Promise.all([
+        pipeline(req, count).catch((e) => {
+          count.destroy(e);
+          throw e;
+        }),
+        runAs.writeAsAsync(member, file, ["replace"], { input: count }),
+      ]);
+    } else {
+      await pipeline(req, count, fs.createWriteStream(file, { mode: 0o600, flags: "wx" }));
    }
-    res.json({ path: file });
-  });
-  req.pipe(out);
-}
-
-router.post(
-  "/drop-file",
-  refuseOversize,
-  streamLocalDrop,
-  express.raw({ type: () => true, limit: DROP_MAX_BYTES }),
-  (req, res) => {
-    const safe = dropName(req);
-    if (!safe || !Buffer.isBuffer(req.body) || req.body.length === 0) {
+    if (bytes === 0) throw Object.assign(new Error("empty"), { code: "EEMPTY" });
+  } catch (err) {
+    try {
+      if (member) runAs.removeAs(member, file);
+      else fs.rmSync(file, { force: true });
+    } catch {
+      /* never written */
+    }
+    // Past the limit mid-body, pipeline has already closed the socket.
+    if (res.headersSent) return;
+    if (err.code === "ETOOBIG") return tooBig(res, max);
+    if (err.code === "EEMPTY") {
      return res.status(400).json({ error: "?name= and a non-empty file body required" });
    }
-    saveDrop(req, res, `${crypto.randomBytes(8).toString("hex")}-${safe}`, req.body);
+    return res.status(500).json({ error: err.message });
  }
-);
+  res.json({ path: file });
+}
+
+router.post("/drop-file", refuseOversize(dropLimit), streamDrop);

 // A browser on this machine is handed a file manager's files as `file://`
 // paths beside their bytes; the composer attaches such a path instead of