chore: v2.8.13 — Artifacts reinstall/edit/installs, agent steps in chat, pick a local file to attach, Windows paths as links

- Chat: an Agent step shows its task, status and tool calls and opens its conversation in place; Windows paths (E:\…, C:/…) in replies open as files
- Prompt: attach a file on this computer by path (native picker), clearer too-large messages
- Artifacts: edit title/description and a skill's files (new snapshot), Reinstall, install history, Hub-only rows, skills installed into ~/.claude/skills and ~/.agents/skills
- Files: batch upload of small files in one extract
- AI Tools: npm install falls back to --prefix ~/.local when the global prefix is not writable
This commit is contained in:
2026-10-20 16:44:55 +07:00
parent 49f87c4da3
commit 0547488f13
59 changed files with 4285 additions and 690 deletions
+12 -7
View File
@@ -2,7 +2,8 @@
* @file The one-at-a-time install/update runner behind Settings → AI Tools.
* Every command comes from a catalog file (never a request) and runs as an
* argv with stdin closed and the login shell's PATH first: `npm install -g
* <pkg>@latest`, a tool's own updater, or a vendor script the server
* <pkg>@latest` (with `--prefix ~/.local` when npm's global prefix is not the
* account's to write — a system-wide Node), a tool's own updater, or a vendor script the server
* downloads (https, same site as the tool's homepage, ≤ 1 MB) and runs with
* `bash <file>`. The job lives in memory — a child of the server, so it dies
* with it — and it reports its end only AFTER the PATH is re-read and the tool
@@ -27,7 +28,8 @@ let jobTimeoutMs = 10 * 60 * 1000;
const SCRIPT_MAX = 1024 * 1024;
const EACCES_HINT =
"\n[agenthub] npm could not write its global directory (EACCES): this Node is installed " +
"system-wide. Use a user-level Node (nvm, fnm) or install the tool by hand.\n";
"system-wide and ~/.local is not writable either. Use a user-level Node (nvm, fnm) or install " +
"the tool by hand.\n";
const EXIT_GRACE_MS = 500; // after 'exit', how long to wait for the pipes to close
const KILL_GRACE_MS = 5000; // SIGTERM → SIGKILL
@@ -44,10 +46,11 @@ function makeErr(code, message) {
return err;
}
const npmArgv = (pkg) => {
/** `prefix`: the `--prefix` args (none for npm's own global prefix). */
const npmArgv = (pkg, prefix = []) => {
const npm = versions.npmBin();
if (!npm) throw makeErr("ENONPM", "npm was not found on the login shell's PATH");
return [npm, "install", "-g", `${pkg}@latest`];
return [npm, "install", "-g", ...prefix, `${pkg}@latest`];
};
/** The steps one action on one tool would run — `{argv}` or `{script: url}`. */
@@ -59,20 +62,22 @@ async function stepsFor(id, action) {
if (!tool.install) throw makeErr("ENOINSTALL", `${tool.label} has no install command`);
if (d.installed) throw makeErr("EINSTALLED", `${tool.label} is already installed`);
return tool.install.kind === "npm"
? [{ argv: npmArgv(tool.install.package) }]
? [{ argv: npmArgv(tool.install.package, await versions.installPrefixArgs()) }]
: [{ script: tool.install.url }];
}
if (!tool.update) throw makeErr("ENOUPDATE", `${tool.label} has no update command`);
if (!d.installed) throw makeErr("ENOTINSTALLED", `${tool.label} is not installed`);
if (tool.update.kind === "self") return [{ argv: [d.path, ...tool.update.args] }];
if (tool.update.kind === "script") return [{ script: tool.install.url }];
if (!(await versions.npmManaged(id))) {
const by = await versions.managedBy(id);
if (!by) {
throw makeErr(
"ENOTNPM",
`${tool.label} was installed outside npm; update it the way it was installed`
);
}
return [{ argv: npmArgv(tool.install.package) }];
const prefix = by === "user" ? ["--prefix", versions.userPrefix()] : [];
return [{ argv: npmArgv(tool.install.package, prefix) }];
}
function append(j, text) {
+78 -5
View File
@@ -5,10 +5,15 @@
* standalone build, say — is never offered an npm update: it would install a
* second copy that the PATH order shadows. npm is the LOGIN SHELL's (its PATH
* first), not the pinned Node's beside the server. Cached for a day.
* A system-wide Node (`/usr/local`, root's) leaves npm's global prefix
* unwritable to the dashboard's account, so there a global install goes to the
* account's own `~/.local` (`--prefix`, bin in `~/.local/bin`) instead, and a
* tool found under `~/.local/lib/node_modules` counts as npm-managed there.
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
*/
const fs = require("node:fs");
const os = require("node:os");
const path = require("node:path");
const { execFile } = require("node:child_process");
const aiTools = require("./ai-tools");
@@ -27,6 +32,8 @@ const defaultExec = (file, args, opts) =>
)
);
let exec = defaultExec;
let userPrefixImpl = () =>
process.platform === "win32" ? null : path.join(os.homedir(), ".local");
/** The PATH a job runs with: the shell's own order first, then the server's. */
const jobPath = () => [shellPath(), process.env.PATH].filter(Boolean).join(path.delimiter);
@@ -62,17 +69,63 @@ function under(prefix, file) {
}
}
/** Whether tool `id` is installed AND lives under npm's global prefix. */
async function npmManaged(id, prefix) {
/** The account's own npm prefix (`~/.local`), or null where npm has none (Windows). */
const userPrefix = () => userPrefixImpl();
/** Whether npm can write `dir` — or, when it is not there yet, its nearest existing parent. */
function writable(dir) {
let d = dir;
for (;;) {
try {
fs.accessSync(d, fs.constants.W_OK);
return true;
} catch (err) {
if (err.code !== "ENOENT") return false;
}
const up = path.dirname(d);
if (up === d) return false;
d = up;
}
}
/** Whether a global install into `prefix` would succeed: its package and bin folders. */
const prefixWritable = (prefix) =>
process.platform === "win32"
? writable(prefix)
: writable(path.join(prefix, "lib", "node_modules")) && writable(path.join(prefix, "bin"));
/**
* The `--prefix` a global install needs: none while npm's own global prefix is
* writable, else the account's `~/.local` (a system-wide Node: EACCES otherwise).
*/
async function installPrefixArgs() {
const npm = npmBin();
const prefix = npm ? await npmPrefix(npm) : null;
const up = userPrefix();
if (!prefix || !up || prefixWritable(prefix)) return [];
return ["--prefix", up];
}
/**
* Which npm prefix holds tool `id`: `"global"` (npm's own), `"user"` (the
* account's `~/.local/lib/node_modules`), or null — not installed, or not by npm.
*/
async function managedBy(id, prefix) {
const d = aiTools.detect(id);
if (!d.installed) return false;
if (!d.installed) return null;
if (prefix === undefined) {
const npm = npmBin();
prefix = npm ? await npmPrefix(npm) : null;
}
return !!prefix && under(prefix, d.path);
if (prefix && under(prefix, d.path)) return "global";
const up = userPrefix();
if (up && under(path.join(up, "lib", "node_modules"), d.path)) return "user";
return null;
}
/** Whether tool `id` is installed AND lives under npm's global or the account's prefix. */
const npmManaged = async (id, prefix) => !!(await managedBy(id, prefix));
async function compute() {
const npm = npmBin();
const prefix = npm ? await npmPrefix(npm) : null;
@@ -114,10 +167,30 @@ function invalidate() {
cache = null;
}
/** Test seam: the account's npm prefix (a path, or null for none); undefined restores it. */
function __setUserPrefix(p) {
userPrefixImpl =
p === undefined
? () => (process.platform === "win32" ? null : path.join(os.homedir(), ".local"))
: () => p;
}
/** Test seam: `fn(file, args, opts) → Promise<{stdout}>`, or null for execFile. */
function __setExec(fn) {
exec = fn || defaultExec;
cache = null;
}
module.exports = { versions, invalidate, npmManaged, npmBin, jobPath, parseVersion, __setExec };
module.exports = {
versions,
invalidate,
npmManaged,
managedBy,
installPrefixArgs,
userPrefix,
npmBin,
jobPath,
parseVersion,
__setExec,
__setUserPrefix,
};
+271 -50
View File
@@ -23,15 +23,21 @@
* their own rows plus shared ones, admin and analyst see every row (like
* sessions), and only the owner (or an admin) changes or deletes one.
* - Installing a shared skill or plugin is the viewer's click, into the
* VIEWER's own account only (`installTarget`): a member's home under S3
* VIEWER's own accounts only (`installAccounts`): a member's home under S3
* (written AS them through `write-as`, `claude` run as them through sudo),
* the host's Claude home for an admin or in local mode. An existing skill
* plus the host's home for an admin (their host-folder workspaces run as
* the host), or the host alone in local mode. A skill goes into BOTH
* `~/.claude/skills/` and the cross-tool `~/.agents/skills/` of each, so
* every AI tool and every session of the viewer finds it. An existing skill
* is never overwritten without `replace`, and the old one is moved aside to
* `~/.claude/agenthub-replaced-skills/`, never deleted. A plugin is
* installed without `-y`, so one whose marketplace declares an install
* command fails rather than runs unseen. An admin can turn installs off
* for everyone (`setInstallsEnabled`). Share, unshare and install are
* written to `audit_log`.
* - Editing a skill's name, description or instructions (`editSkill`) writes
* a NEW snapshot and swaps the row to it; the old snapshot is dropped, and
* copies already installed stay as they were (Reinstall picks it up).
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
*/
@@ -133,6 +139,7 @@ function view(row, req, owners = null) {
size: row.size,
shared: row.shared === 1,
sharedAt: row.shared_at,
inLibrary: row.in_library !== 0,
installs: row.installs,
createdAt: row.created_at,
updatedAt: row.updated_at,
@@ -142,13 +149,14 @@ function view(row, req, owners = null) {
/**
* The rows this request may see. `scope`: `mine` (own rows; every row in
* local mode), `hub` (shared rows, server mode), anything else = all visible.
* Each row also carries `installed` (`installedCheck`).
*/
function list(req, { scope = "all", kind, q, laneId, sessionId, limit = 200 } = {}) {
const v = visibleWhere(req);
const where = [v.where];
const params = [...v.params];
if (scope === "mine" && req?.user) {
where.push("a.owner_id = ?");
where.push("a.owner_id = ? AND a.in_library = 1");
params.push(req.user.id);
} else if (scope === "hub") {
where.push("a.shared = 1");
@@ -182,7 +190,8 @@ function list(req, { scope = "all", kind, q, laneId, sessionId, limit = 200 } =
`SELECT a.* FROM artifacts a WHERE ${where.join(" AND ")} ORDER BY a.updated_at DESC, a.id DESC LIMIT ${n}`
).all(...params);
const owners = new Map();
return rows.map((r) => view(r, req, owners));
const installed = installedCheck(req);
return rows.map((r) => ({ ...view(r, req, owners), installed: installed(r) }));
}
// ── Small helpers ─────────────────────────────────────────────────────────
@@ -255,7 +264,7 @@ function addPage(req, { url, title, description, laneId, sessionId }) {
);
if (same) {
sql(
"UPDATE artifacts SET title = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') WHERE id = ?"
"UPDATE artifacts SET title = ?, in_library = 1, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') WHERE id = ?"
).run(cleanTitle(title, same.title), same.id);
return view(sql("SELECT * FROM artifacts WHERE id = ?").get(same.id), req);
}
@@ -1114,7 +1123,7 @@ function addPlugin(req, { key, market, title, description }) {
).get(k, ownerId);
if (same) {
sql(
`UPDATE artifacts SET market = ?, title = ?, description = ?,
`UPDATE artifacts SET market = ?, title = ?, description = ?, in_library = 1,
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') WHERE id = ?`
).run(
src,
@@ -1154,8 +1163,14 @@ function audit(req, row, action) {
}
function update(req, id, body = {}) {
const row = load(req, id);
let row = load(req, id);
if (!canChange(req, row)) throw fail("EFORBIDDEN", "only its owner changes an artifact");
if (body.skill && typeof body.skill === "object") {
editSkill(req, row, body.skill);
row = sql("SELECT * FROM artifacts WHERE id = ?").get(row.id);
if (!("title" in body) && !("shared" in body) && !("description" in body))
return view(row, req);
}
const sets = [];
const params = [];
if ("title" in body) {
@@ -1187,6 +1202,12 @@ function update(req, id, body = {}) {
}
}
if (!sets.length) return view(row, req);
if (shareChange === "unshared" && row.in_library === 0) {
// Off the Hub and out of its owner's library: nothing keeps it.
deleteRow(row);
audit(req, row, "unshared");
return { ...view(row, req), shared: false, removed: true };
}
sets.push("updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')");
sql(`UPDATE artifacts SET ${sets.join(", ")} WHERE id = ?`).run(...params, row.id);
const next = sql("SELECT * FROM artifacts WHERE id = ?").get(row.id);
@@ -1194,15 +1215,86 @@ function update(req, id, body = {}) {
return view(next, req);
}
function deleteRow(row) {
sql("DELETE FROM artifact_installs WHERE artifact_id = ?").run(row.id);
sql("DELETE FROM artifacts WHERE id = ?").run(row.id);
dropSnapshot(row.snapshot);
}
/**
* Delete from the caller's library. A SHARED row its owner deletes stays in
* the Hub (`in_library = 0`, `{kept: true}`) — only unsharing takes it off
* there. Anything else (unshared, already Hub-only, or an admin removing
* someone else's) is deleted with its snapshot.
*/
function remove(req, id) {
const row = load(req, id);
if (!canChange(req, row)) throw fail("EFORBIDDEN", "only its owner deletes an artifact");
sql("DELETE FROM artifacts WHERE id = ?").run(row.id);
dropSnapshot(row.snapshot);
const owner = !req?.user || row.owner_id === req.user.id;
if (owner && row.shared === 1 && row.in_library !== 0) {
sql(
"UPDATE artifacts SET in_library = 0, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') WHERE id = ?"
).run(row.id);
return { ...row, in_library: 0, kept: true };
}
deleteRow(row);
if (row.shared === 1) audit(req, row, "deleted");
return row;
}
/**
* New contents for a skill row: `name` (the folder and /command),
* `description` (frontmatter — also the card's description) and `content`
* (SKILL.md's body, frontmatter excluded). The other frontmatter keys and
* every other file are kept. A new snapshot replaces the old one.
*/
function editSkill(req, row, { name, description, content }) {
if (row.kind !== "skill") throw fail("EBADKIND", "only a skill's contents are edited");
if (!row.snapshot || !SNAPSHOT_RE.test(row.snapshot)) throw fail("ENOFILE", "no files");
const n = validSkillName(name);
const body = typeof content === "string" ? content : "";
if (!body.trim()) throw fail("EBADSKILL", "write the skill's instructions");
const recorded = row.files ? JSON.parse(row.files) : [];
const files = recorded.map((f) => ({
path: f.path,
data: fs.readFileSync(path.join(root(), row.snapshot, ...f.path.split("/"))),
}));
const md = files.find((f) => f.path === "SKILL.md");
const old = md ? md.data.toString("utf8").replace(/^\uFEFF/, "") : "";
const fm = /^---\r?\n[\s\S]*?\r?\n---[^\n]*(\r?\n|$)/.exec(old);
const text = withFrontmatter(
`${fm ? fm[0] : ""}${fm ? "\n" : ""}${body.replace(/^\s+/, "")}`,
n,
description
);
const all = [
{ path: "SKILL.md", data: Buffer.from(text, "utf8") },
...files.filter((f) => f !== md),
];
checkSkillFiles(all);
const snap = newSnapshotDir();
let total = 0;
const next = [];
try {
for (const f of all) {
const target = path.join(snap.dir, ...f.path.split("/"));
fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 });
fs.writeFileSync(target, f.data, { mode: 0o600 });
total += f.data.length;
next.push({ path: f.path, size: f.data.length });
}
} catch (err) {
dropSnapshot(snap.id);
throw err;
}
const desc = frontmatterField(text, "description");
sql(
`UPDATE artifacts SET source = ?, snapshot = ?, files = ?, size = ?, description = ?,
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') WHERE id = ?`
).run(n, snap.id, JSON.stringify(next), total, cleanDescription(desc), row.id);
dropSnapshot(row.snapshot);
}
/** The absolute path of one RECORDED file of a row's snapshot (`rel` must match). */
function snapshotFile(req, id, rel) {
const row = load(req, id);
@@ -1237,55 +1329,140 @@ function installable(req, row) {
if (!installsEnabled()) throw fail("EINSTALLSOFF", "an admin turned Hub installs off");
}
/** Install a skill snapshot into the viewer's own `~/.claude/skills/<name>`. */
async function installSkill(req, row, { replace = false } = {}) {
const { acct, claudeHome } = ownAccount(req);
const runAs = require("./run-as");
const skills = path.join(claudeHome, "skills");
const target = path.join(skills, row.source);
const taken = (() => {
try {
fs.lstatSync(target);
return true;
} catch {
return false;
}
})();
if (taken && !replace) throw fail("EEXISTS", `a skill named ${row.source} is already installed`);
if (taken) {
const aside = path.join(
claudeHome,
"agenthub-replaced-skills",
`${row.source}-${new Date().toISOString().replace(/[:.]/g, "-")}`
);
if (acct) {
if (!runAs.writeAsCanMove()) {
throw fail(
"EUPGRADE",
"replacing needs the updated helper: run `agenthub server install-system` once"
);
/**
* Every account a request's sessions run as: `[{acct, home, claudeHome}]`.
* Under S3 a pane runs as its FOLDER's account, so an admin with a member
* account of their own also runs host-folder workspaces as the host — both
* get the skill, or half their sessions never see it.
*/
function installAccounts(req) {
const hostHome = path.dirname(getClaudeHome());
const host = { acct: null, home: hostHome, claudeHome: getClaudeHome() };
if (!req?.user) return [host];
const own = ownAccount(req);
const out = [];
if (own.acct) out.push({ acct: own.acct, home: own.acct.home, claudeHome: own.claudeHome });
if (req.user.role === "admin") out.push(host);
return out;
}
/**
* Where an account's AI tools look for user skills: Claude Code's
* `~/.claude/skills` (also read by OpenCode, Cursor, omp, Grok) and the
* cross-tool `~/.agents/skills` (Codex, Gemini/Antigravity, OpenCode, Cursor).
*/
const skillDirsOf = (a) => [
path.join(a.claudeHome, "skills"),
path.join(a.home, ".agents", "skills"),
];
const exists = (p) => {
try {
fs.lstatSync(p);
return true;
} catch {
return false;
}
};
/**
* `row => boolean`: is this skill/plugin already in every account the
* viewer's sessions run as (`installAccounts`) — a skill as
* `~/.claude/skills/<name>`, a plugin as a key of `installed_plugins.json`.
* The list shows "Installed" instead of Install for those. Pure reads; an
* account that cannot be resolved or read counts as not installed.
*/
function installedCheck(req) {
let accounts = null;
const plugins = new Map();
const pluginKeys = (a) => {
if (!plugins.has(a.claudeHome)) {
const file = path.join(a.claudeHome, "plugins", "installed_plugins.json");
let keys = new Set();
try {
const text = a.acct
? require("./run-as").readFileSafe(file, "utf8", { max: 4 << 20 })
: fs.readFileSync(file, "utf8");
keys = new Set(Object.keys(JSON.parse(text)?.plugins || {}));
} catch {
/* none installed, or unreadable */
}
runAs.mkdirAs(acct, path.dirname(aside));
await runAs.writeAsAsync(acct, target, ["move", aside]);
plugins.set(a.claudeHome, keys);
}
return plugins.get(a.claudeHome);
};
return (row) => {
if ((row.kind !== "skill" && row.kind !== "plugin") || !row.source) return false;
if (accounts === null) {
try {
accounts = installAccounts(req);
} catch {
accounts = [];
}
}
if (!accounts.length) return false;
return accounts.every((a) =>
row.kind === "skill"
? exists(path.join(a.claudeHome, "skills", row.source))
: pluginKeys(a).has(row.source)
);
};
}
/**
* Install a skill snapshot as `<name>` into every skills folder of every
* account the viewer's sessions run as (`installAccounts` × `skillDirsOf`).
*/
async function installSkill(req, row, { replace = false } = {}) {
const runAs = require("./run-as");
const targets = installAccounts(req).flatMap((a) =>
skillDirsOf(a).map((dir) => ({ ...a, target: path.join(dir, row.source) }))
);
const taken = targets.filter((t) => exists(t.target));
if (taken.length && !replace) {
throw fail("EEXISTS", `a skill named ${row.source} is already installed`);
}
if (taken.some((t) => t.acct) && !runAs.writeAsCanMove()) {
throw fail(
"EUPGRADE",
"replacing needs the updated helper: run `agenthub server install-system` once"
);
}
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
for (const t of taken) {
const tag = t.target.startsWith(t.claudeHome + path.sep) ? "" : "-agents";
const aside = path.join(
t.claudeHome,
"agenthub-replaced-skills",
`${row.source}${tag}-${stamp}`
);
if (t.acct) {
runAs.mkdirAs(t.acct, path.dirname(aside));
await runAs.writeAsAsync(t.acct, t.target, ["move", aside]);
} else {
fs.mkdirSync(path.dirname(aside), { recursive: true });
fs.renameSync(target, aside);
fs.renameSync(t.target, aside);
}
}
const files = row.files ? JSON.parse(row.files) : [];
for (const f of files) {
const from = path.join(root(), row.snapshot, ...f.path.split("/"));
const to = path.join(target, ...f.path.split("/"));
const content = fs.readFileSync(from);
if (acct) {
runAs.mkdirAs(acct, path.dirname(to));
runAs.writeFileAs(to, content);
} else {
fs.mkdirSync(path.dirname(to), { recursive: true });
fs.writeFileSync(to, content);
const contents = files.map((f) => ({
rel: f.path.split("/"),
content: fs.readFileSync(path.join(root(), row.snapshot, ...f.path.split("/"))),
}));
for (const t of targets) {
for (const f of contents) {
const to = path.join(t.target, ...f.rel);
if (t.acct) {
runAs.mkdirAs(t.acct, path.dirname(to));
runAs.writeFileAs(to, f.content);
} else {
fs.mkdirSync(path.dirname(to), { recursive: true });
fs.writeFileSync(to, f.content);
}
}
}
return { installed: target, replaced: taken };
const paths = targets.map((t) => t.target);
return { installed: paths.join(", "), paths, replaced: taken.length > 0 };
}
let execImpl = promisify(require("node:child_process").execFile);
@@ -1343,12 +1520,56 @@ async function install(req, id, opts = {}) {
const result =
row.kind === "skill" ? await installSkill(req, row, opts) : await installPlugin(req, row);
sql("UPDATE artifacts SET installs = installs + 1 WHERE id = ?").run(row.id);
sql(
"INSERT INTO artifact_installs (artifact_id, user_id, paths, replaced) VALUES (?, ?, ?, ?)"
).run(row.id, req?.user?.id ?? null, JSON.stringify(result.paths || []), result.replaced ? 1 : 0);
if (req?.user && row.owner_id !== req.user.id) audit(req, row, "installed");
return result;
}
/**
* Who installed artifact `id`, newest first: `{items: [{user, mine, at,
* replaced, paths?}], unrecorded}`. The owner, staff and local mode see every
* install; anyone else only their own. `paths` (where a skill was written; a
* plugin has none — it is `claude plugin install --scope user`) goes only to
* the installer and an admin. `unrecorded` = installs counted before this
* list was kept.
*/
function installsOf(req, id) {
const row = load(req, id);
const u = req?.user;
const all = !u || u.role === "admin" || u.role === "analyst" || row.owner_id === u.id;
const rows = sql(
`SELECT i.*, us.username, us.display_name FROM artifact_installs i
LEFT JOIN users us ON us.id = i.user_id
WHERE i.artifact_id = ? ${all ? "" : "AND i.user_id = ?"}
ORDER BY i.id DESC LIMIT 500`
).all(...(all ? [row.id] : [row.id, u.id]));
const items = rows.map((r) => {
const mine = !u || r.user_id === u.id;
let paths = [];
try {
paths = JSON.parse(r.paths) || [];
} catch {
paths = [];
}
return {
user: r.user_id == null ? null : r.display_name || r.username || null,
mine,
at: r.installed_at,
replaced: r.replaced === 1,
...(mine || u?.role === "admin" ? { paths } : {}),
};
});
const total = sql("SELECT COUNT(*) AS n FROM artifact_installs WHERE artifact_id = ?").get(
row.id
).n;
return { items, unrecorded: all ? Math.max(0, (row.installs || 0) - total) : 0 };
}
module.exports = {
KINDS,
installsOf,
DOC_EXT,
list,
load,
+351 -2
View File
@@ -4,7 +4,8 @@
* or into another of the same account), delete into a per-account TRASH that
* can be restored (kept TRASH_DAYS) — and, for `/api/homes`, one recycle bin
* per USER over every store their deletes went to — upload a file streamed from the request
* into any (new) sub-folder, and download files or folders as one streamed zip.
* into any (new) sub-folder (or many small ones as ONE batch request), and
* download files or folders as one streamed zip.
*
* Every path is the read path's own (git-diff.js): `lanePath` (lexical, no
* `..`), the PARENT resolved by `resolveInLane` (realpath-confined, so no
@@ -37,7 +38,8 @@ const crypto = require("node:crypto");
const fs = require("node:fs");
const path = require("node:path");
const zlib = require("node:zlib");
const { Readable, Transform } = require("node:stream");
const { PassThrough, Readable, Transform } = require("node:stream");
const { once } = require("node:events");
const { AsyncLocalStorage } = require("node:async_hooks");
const { pipeline } = require("node:stream/promises");
@@ -706,6 +708,351 @@ async function upload(dir, rel, stream, { conflict, length, trash } = {}) {
return { path: loc.rel };
}
// ── Batch upload ───────────────────────────────────────────────────────────
//
// Many small files cost per FILE, not per byte: one request, one path walk
// and — in a member's folder — one sudo write-as each. A batch carries them
// in ONE request: a 4-byte big-endian length, that many bytes of JSON
// `{files: [{path, size, mtime?}]}` (paths relative to `?dir=`), then each
// file's bytes in that order. Every name is checked like `upload`'s and the
// whole batch is planned (folders, conflicts) before the first write; a
// member's batch is written by ONE `write-as <dir> extract` from a tar this
// module writes itself (only folders and regular files, names it checked).
const BATCH_MAX_FILES = 5000;
/** A file in a batch is held in memory whole; bigger ones go through `upload`. */
const BATCH_FILE_MAX = 16 * 1024 * 1024;
const BATCH_MAX_BYTES = 256 * 1024 * 1024;
const BATCH_HEAD_MAX = 4 * 1024 * 1024;
/** Whether a member's batch can be written (write-as `extract`, install-system re-run). */
function needExtract(acct) {
if (acct && !runAs.writeAsCanExtract()) {
throw bad(
"EUPGRADE",
"uploading many files at once into a member's folder needs the updated helper: run `agenthub server install-system` once"
);
}
}
/** Reads exact byte counts from a stream (the request body). */
function byteReader(stream) {
const it = stream[Symbol.asyncIterator]();
let chunks = [];
let have = 0;
return {
async read(n) {
while (have < n) {
const { value, done } = await it.next();
if (done) throw bad("EBADBODY", "the upload ended early");
const b = Buffer.isBuffer(value) ? value : Buffer.from(value);
chunks.push(b);
have += b.length;
}
const all = chunks.length === 1 ? chunks[0] : Buffer.concat(chunks, have);
const rest = all.subarray(n);
chunks = rest.length ? [rest] : [];
have = rest.length;
return all.subarray(0, n);
},
/** Consume whatever is left, so an answer can still be sent. */
async drain() {
chunks = [];
have = 0;
for (;;) if ((await it.next()).done) return;
},
};
}
/** The `files` list of a batch, each `{path, size, mtime}` checked for shape. */
function batchFiles(list) {
if (!Array.isArray(list) || !list.length) throw bad("EBADBODY", "a batch names its files");
if (list.length > BATCH_MAX_FILES) {
throw bad("ETOOBIG", `a batch holds at most ${BATCH_MAX_FILES} files`);
}
let total = 0;
return list.map((f) => {
const size = f?.size ?? 0;
if (typeof f?.path !== "string" || !Number.isSafeInteger(size) || size < 0) {
throw bad("EBADBODY", "each file is {path, size}");
}
if (size > BATCH_FILE_MAX) {
throw bad("ETOOBIG", `a file in a batch may be at most ${BATCH_FILE_MAX / 1024 / 1024} MB`);
}
total += size;
if (total > BATCH_MAX_BYTES) {
throw bad("ETOOBIG", `a batch may be at most ${BATCH_MAX_BYTES / 1024 / 1024} MB`);
}
const mtime = Number.isFinite(f.mtime) && f.mtime > 0 ? f.mtime : Date.now();
return { path: f.path, size, mtime };
});
}
/**
* Where each file of a batch goes, under `conflict`: the folders to make
* (parents first) and per file its target, or `skip`/`replace`. `taken`
* lists the names already there (paths relative to the batch's folder);
* under `fail` they are not resolved — the caller refuses or reports them.
*/
async function planBatch(dir, dirRel, files, conflict) {
const base = await folder(dir, dirRel || "");
if (gitInside(base.rel)) throw bad("EBADPATH", "nothing is uploaded into .git");
const full = (r) => (base.rel ? `${base.rel}/${r}` : r);
// rel under base → {rel (workspace), abs, missing}
const dirs = new Map([["", { rel: base.rel, abs: base.abs, missing: false }]]);
const make = [];
const dirOf = async (r) => {
if (dirs.has(r)) return dirs.get(r);
const upRel = path.posix.dirname(r) === "." ? "" : path.posix.dirname(r);
const up = await dirOf(upRel);
const name = path.posix.basename(r);
const d = { rel: full(r), abs: path.join(up.abs, name), missing: true };
if (up.missing) {
if (topDot(base.root, d.abs)) throw hidden();
if (await isSecretOnDisk(dir, d.rel)) throw bad("ESECRET", "that path holds secrets");
} else {
const loc = await locate(dir, d.rel);
if (loc.st && !loc.st.isDirectory()) {
throw bad("EEXISTS", `${d.rel} is a file, not a folder`, { names: [r] });
}
d.missing = !loc.st;
}
if (d.missing) make.push(d);
dirs.set(r, d);
return d;
};
const seen = new Set();
// A free name for `keep` is free on disk AND not another file of this batch.
const claimed = new Set();
const own = new Set(
files.map((f) => {
try {
return lanePath(f.path);
} catch {
return "";
}
})
);
const out = [];
const taken = [];
for (const f of files) {
const r = lanePath(f.path);
for (const seg of r.split("/")) validName(seg);
if (gitInside(full(r))) throw bad("EBADPATH", "nothing is uploaded into .git");
if (seen.has(r)) throw bad("EBADPATH", `${r} is named twice`);
seen.add(r);
const upRel = path.posix.dirname(r) === "." ? "" : path.posix.dirname(r);
const up = await dirOf(upRel);
let name = path.posix.basename(r);
let st = null;
if (up.missing) {
if (await isSecretOnDisk(dir, full(r))) throw bad("ESECRET", "that path holds secrets");
} else {
const loc = await locate(dir, full(r));
st = loc.st;
}
const step = { ...f, rel: r, name, up, skip: false, replace: false };
if (st) {
if (st.isDirectory()) throw bad("EEXISTS", `${full(r)} is a folder`, { names: [r] });
taken.push(r);
if (conflict === "skip") step.skip = true;
else if (conflict === "replace") step.replace = true;
else if (conflict === "keep") {
const ext = name.startsWith(".") ? "" : path.extname(name);
const stem = ext ? name.slice(0, -ext.length) : name;
for (let i = 1; ; i++) {
if (i >= 10_000) throw bad("EEXISTS", `no free name for ${name}`);
const n = `${stem} (${i})${ext}`;
const abs = path.join(up.abs, n);
const nRel = upRel ? `${upRel}/${n}` : n;
if (claimed.has(abs) || own.has(nRel)) continue;
if (await fs.promises.lstat(abs).catch(() => null)) continue;
name = n;
break;
}
}
}
step.name = name;
step.abs = path.join(up.abs, name);
step.full = up.rel ? `${up.rel}/${name}` : name;
claimed.add(step.abs);
out.push(step);
}
for (const r of seen) {
if (dirs.has(r)) throw bad("EBADPATH", `${r} is both a file and a folder`);
}
return { base, make, files: out, taken };
}
/**
* `{taken, batch}` for a batch about to be sent: the names already in
* `dirRel` (relative to it), and whether a batch can be written there at all
* (false: send the files one by one).
*/
async function uploadCheck(dir, dirRel, paths) {
if (!Array.isArray(paths) || !paths.every((p) => typeof p === "string")) {
throw bad("EBADBODY", "paths is a list of relative paths");
}
const files = batchFiles(paths.map((p) => ({ path: p, size: 0 })));
const plan = await planBatch(dir, dirRel, files, "fail");
let batch = true;
try {
needExtract(accountAt(plan.base.abs));
} catch {
batch = false;
}
return { taken: plan.taken, batch };
}
/** One octal tar field of `len` bytes. */
function tarOctal(buf, off, len, n) {
buf.write(`${n.toString(8).padStart(len - 1, "0")}\0`, off, len, "ascii");
}
/** A ustar header block for `name` (a pax `path` record first when it is long). */
function tarHeader(name, { type, size, mode, mtime }) {
const block = (n, t, sz) => {
const h = Buffer.alloc(512);
h.write(n, 0, 100, "utf8");
tarOctal(h, 100, 8, mode);
tarOctal(h, 108, 8, 0);
tarOctal(h, 116, 8, 0);
tarOctal(h, 124, 12, sz);
tarOctal(h, 136, 12, Math.floor(mtime / 1000));
h.fill(" ", 148, 156);
h.write(t, 156, 1, "ascii");
h.write("ustar\0" + "00", 257, 8, "ascii");
let sum = 0;
for (const b of h) sum += b;
h.write(`${sum.toString(8).padStart(6, "0")}\0 `, 148, 8, "ascii");
return h;
};
if (Buffer.byteLength(name) <= 100) return block(name, type, size);
// "<len> path=<name>\n", where <len> counts its own digits.
const body = ` path=${name}\n`;
let len = Buffer.byteLength(body);
while (String(len).length + Buffer.byteLength(body) !== len) {
len = String(len).length + Buffer.byteLength(body);
}
const rec = Buffer.from(`${len}${body}`, "utf8");
return Buffer.concat([block("PaxHeader", "x", rec.length), tarPad(rec), block("", type, size)]);
}
function tarPad(buf) {
const rest = buf.length % 512;
return rest ? Buffer.concat([buf, Buffer.alloc(512 - rest)]) : buf;
}
/**
* Write a batch streamed from `stream` (see above) into `dirRel`. Returns
* `{saved, skipped, paths}`; under `conflict: "fail"` a taken name is 409
* EEXISTS with every taken path in `names`.
*/
async function uploadBatch(dir, dirRel, stream, { conflict, trash } = {}) {
const reader = byteReader(stream);
try {
const n = (await reader.read(4)).readUInt32BE(0);
if (n > BATCH_HEAD_MAX) throw bad("ETOOBIG", "a batch's file list is too long");
let head;
try {
head = JSON.parse((await reader.read(n)).toString("utf8"));
} catch (e) {
if (e.code) throw e;
throw bad("EBADBODY", "a batch starts with its file list as JSON");
}
const files = batchFiles(head?.files);
const policy = conflictOf(conflict);
const plan = await planBatch(dir, dirRel, files, policy);
if (policy === "fail" && plan.taken.length) {
throw bad("EEXISTS", `${plan.taken.length} name(s) already exist`, { names: plan.taken });
}
const acct = accountAt(plan.base.abs);
needExtract(acct);
for (const f of plan.files) {
if (f.replace) await trash(f.full);
}
const done = [];
let skipped = 0;
if (acct) {
const tar = new PassThrough();
const put = async (buf) => {
// write-as gone (it failed): a destroyed stream never drains.
if (tar.destroyed) throw bad("EWRITEAS", "the batch could not be written");
if (!tar.write(buf)) await once(tar, "drain");
};
const now = Date.now();
const extract = runAs.writeAsAsync(acct, plan.base.abs, ["extract"], { input: tar });
const feed = (async () => {
for (const d of plan.make) {
const name = `${path.relative(plan.base.abs, d.abs).split(path.sep).join("/")}/`;
await put(tarHeader(name, { type: "5", size: 0, mode: 0o755, mtime: now }));
}
for (const f of plan.files) {
const data = await reader.read(f.size);
if (f.skip) {
skipped++;
continue;
}
const name = path.relative(plan.base.abs, f.abs).split(path.sep).join("/");
await put(tarHeader(name, { type: "0", size: f.size, mode: 0o644, mtime: f.mtime }));
await put(tarPad(data));
done.push(f.full);
}
tar.end(Buffer.alloc(1024));
})();
await Promise.all([
feed.catch((e) => {
tar.destroy(e);
throw e;
}),
extract.catch((e) => {
tar.destroy(e);
throw e;
}),
]);
} else {
for (const d of plan.make) {
await fs.promises.mkdir(d.abs).catch(async (e) => {
const st = e.code === "EEXIST" && (await fs.promises.lstat(d.abs).catch(() => null));
if (!st?.isDirectory()) throw e;
});
}
for (const f of plan.files) {
const data = await reader.read(f.size);
if (f.skip) {
skipped++;
continue;
}
const tmp = path.join(
f.up.abs,
`.${f.name}.agenthub-upload-${crypto.randomBytes(4).toString("hex")}`
);
try {
await fs.promises.writeFile(tmp, data, { flag: "wx" });
const t = new Date(f.mtime);
await fs.promises.utimes(tmp, t, t);
// Linked, not renamed: a name taken meanwhile is refused, never replaced.
await fs.promises.link(tmp, f.abs).catch(async (e) => {
if (e.code === "EEXIST") {
throw bad("EEXISTS", `${f.rel} already exists`, { names: [f.rel] });
}
if (e.code === "EPERM" || e.code === "ENOTSUP") return fs.promises.rename(tmp, f.abs);
throw e;
});
} finally {
await fs.promises.rm(tmp, { force: true });
}
done.push(f.full);
}
}
return { saved: done.length, skipped, paths: done };
} catch (e) {
await reader.drain().catch(() => {});
throw e;
}
}
/**
* One file for a download (the Files tab's `openDownload`, after this
* module's own checks — the home rule included, on the file's REAL path).
@@ -893,6 +1240,8 @@ module.exports = {
restore,
purgeTrash,
upload,
uploadBatch,
uploadCheck,
zip,
collect,
UPLOAD_MAX_BYTES,
+4 -2
View File
@@ -866,8 +866,10 @@ async function cloneNow({ url, parent, name, once = null, partial = false }, thr
const account = accountForDir(parent);
const remote = remoteOf(url);
// index-pack's delta threads: this clone's share of the cores, not all of
// them (clone-queue.js). Not persisted — a later fetch keeps git's default.
const pre = ["-c", `pack.threads=${threads}`];
// them (clone-queue.js) — and as many checkout workers: a tree of many
// small files is written one file at a time otherwise (~4x slower on 4
// cores). Not persisted — a later fetch or checkout keeps git's default.
const pre = ["-c", `pack.threads=${threads}`, "-c", `checkout.workers=${threads}`];
const persist = [];
const env = {};
let restore = null;
+10
View File
@@ -301,6 +301,15 @@ function writeAsCanMove() {
}
}
/** Whether the installed write-as knows `extract` (a batch upload; install-system re-run). */
function writeAsCanExtract() {
try {
return /^\s*extract\)/m.test(fs.readFileSync(WRITE_AS, "utf8"));
} catch {
return false;
}
}
/**
* Save `content` to `file` — as the member who owns the folder (S3), else
* directly. In place, keeping the file's mode; `append` adds to the end.
@@ -634,6 +643,7 @@ module.exports = {
writeAs,
writeAsAsync,
writeAsCanMove,
writeAsCanExtract,
__setSpawnImpl,
mkdirAs,
makeMemberFolder,