Merge branch 'perf/local-opt': faster local install and updates, lighter polling and ingest

scripts/import-history.js

@@ -1937,6 +1937,47 @@ async function ingestWorkflowsForBatch(dbModule, sessions) {
   }
 }
 
+// Session sync: a session counts as fed LIVE by hooks while it is active, its
+// newest event came from a hook (not this importer), and a hook touched it
+// within this window. Such a transcript is left to the hooks until it goes
+// quiet. DASHBOARD_SYNC_LIVE_HOOK_MS=0 turns the deferral off.
+const LIVE_HOOK_MS = (() => {
+  const n = Number(process.env.DASHBOARD_SYNC_LIVE_HOOK_MS ?? 5 * 60_000);
+  return Number.isFinite(n) && n >= 0 ? n : 5 * 60_000;
+})();
+// mtimeCache value for a deferred file: present (so the fs watcher ignores its
+// appends) yet never equal to a real mtime (so every sweep re-checks it).
+const DEFERRED_MTIME = -1;
+const IMPORTED_DATA_RE = /"imported"\s*:\s*true/;
+const latestEventStmts = new WeakMap();
+
+function rowIsImported(row) {
+  try {
+    return !!(row.metadata && JSON.parse(row.metadata).imported);
+  } catch {
+    return false;
+  }
+}
+
+function isHookLive(dbModule, row) {
+  if (!(LIVE_HOOK_MS > 0) || row.status !== "active") return false;
+  const touched = Date.parse(row.updated_at);
+  if (!Number.isFinite(touched) || Date.now() - touched > LIVE_HOOK_MS) return false;
+  try {
+    let stmt = latestEventStmts.get(dbModule.db);
+    if (!stmt) {
+      stmt = dbModule.db.prepare(
+        "SELECT data FROM events WHERE session_id = ? ORDER BY id DESC LIMIT 1"
+      );
+      latestEventStmts.set(dbModule.db, stmt);
+    }
+    const ev = stmt.get(row.id);
+    return !!ev && !IMPORTED_DATA_RE.test(ev.data || "");
+  } catch {
+    return false; // when in doubt, sync as before
+  }
+}
+
 /**
  * Incremental, change-driven sync of the default `~/.claude/projects` directory.
  *
@@ -2024,6 +2065,25 @@ async function syncDefaultProjects(dbModule, options = {}) {
             continue;
           }
         }
+        // A session hooks are feeding RIGHT NOW: its transcript keeps growing
+        // between hooks, so `mtime > updated_at` on nearly every sweep, and a
+        // full re-parse + snapshot copy here bought nothing the hooks do not
+        // already record. Leave it until it goes quiet (or ends): the sentinel
+        // keeps the watcher ignoring its appends, and never equals a real
+        // mtime, so the first sweep after the hooks stop catches it up.
+        if (existingRow && isHookLive(dbModule, existingRow)) {
+          mtimeCache.set(sourcePath, DEFERRED_MTIME);
+          continue;
+        }
+        // A session a hook created (no `imported` flag): importSession() skips
+        // it outright, so parsing the whole transcript + subagents was pure
+        // waste — only the durable snapshot copy still matters.
+        if (existingRow && !rowIsImported(existingRow)) {
+          mtimeCache.set(sourcePath, mtime);
+          if (require("../server/lib/run-as").sameOwner(sourcePath, existingRow.cwd))
+            await snapshotTranscript(sourcePath, sessionId);
+          continue;
+        }
         const existed = !!existingRow;
         const session = await parseSessionForImport(projPath, sourcePath);
         // Record the mtime even when the file yields no session, so we don't

scripts/plugin-bootstrap.js

@@ -138,14 +138,38 @@ function serverIsLive() {
   return livePids().length > 0;
 }
 
+/**
+ * What npm installs from a manifest: its dependency lists, or the lockfile's
+ * package tree — never the release's own `version`, which both files carry.
+ * Hashing the whole files reinstalled every dependency on each update (minutes
+ * on a slow link) when the bump changed nothing npm would fetch.
+ */
+function installedPart(file, text) {
+  const json = JSON.parse(text);
+  if (file === "package.json") {
+    const { dependencies, optionalDependencies, overrides } = json;
+    return JSON.stringify({ dependencies, optionalDependencies, overrides });
+  }
+  const { "": own, ...packages } = json.packages || {};
+  const { version: _version, ...rest } = own || {};
+  return JSON.stringify({ own: rest, packages });
+}
+
 /** Hash of the dependency manifest — changes mean the runtime tree must be reinstalled. */
 function depsHash(root = PLUGIN_ROOT) {
   const h = crypto.createHash("sha256");
   for (const f of ["package.json", "package-lock.json"]) {
+    let text;
     try {
-      h.update(fs.readFileSync(path.join(root, f)));
+      text = fs.readFileSync(path.join(root, f), "utf8");
     } catch {
       h.update(f); // absent counts as its own state
+      continue;
+    }
+    try {
+      h.update(installedPart(f, text));
+    } catch {
+      h.update(text); // not JSON npm could read either: the bytes decide
     }
   }
   return h.digest("hex").slice(0, 16);
@@ -196,6 +220,81 @@ function nodeVersionOk(version = process.versions.node) {
   return true;
 }
 
+/* ------------------------------------------------------- last failure report */
+
+/**
+ * The worker runs detached with its output discarded, so a failed install, a
+ * Node that is too old or a failed UI build used to live only in a log nobody
+ * opens — and every session silently retried. The last failure is kept in one
+ * small file; the NEXT foreground pass (which Claude Code reads) prints it once
+ * as a SessionStart `systemMessage`, and a bootstrap that succeeds removes it.
+ */
+function failurePath(rt = runtimeDir()) {
+  return path.join(rt, "last-failure.json");
+}
+
+function readFailure(rt = runtimeDir()) {
+  try {
+    return JSON.parse(fs.readFileSync(failurePath(rt), "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+/** Record a failure; the same kind again keeps its `reported` flag (once per failure). */
+function recordFailure(rt, kind, message, logFile) {
+  const prev = readFailure(rt);
+  const same = prev && prev.kind === kind && prev.message === message;
+  const rec = {
+    kind,
+    message,
+    log: logFile,
+    at: new Date().toISOString(),
+    reported: Boolean(same && prev.reported),
+  };
+  try {
+    fs.mkdirSync(rt, { recursive: true });
+    fs.writeFileSync(failurePath(rt), JSON.stringify(rec, null, 2) + "\n", "utf8");
+  } catch {
+    /* best effort */
+  }
+}
+
+function clearFailure(rt = runtimeDir()) {
+  try {
+    fs.rmSync(failurePath(rt), { force: true });
+  } catch {
+    /* best effort */
+  }
+}
+
+/**
+ * The SessionStart output for an unreported failure, or null. Marks it
+ * reported, so a failure is shown once — not on every session until fixed.
+ */
+function takeFailureReport(rt = runtimeDir()) {
+  const rec = readFailure(rt);
+  if (!rec || rec.reported) return null;
+  try {
+    fs.writeFileSync(failurePath(rt), JSON.stringify({ ...rec, reported: true }, null, 2) + "\n");
+  } catch {
+    return null; // could not mark it: stay quiet rather than repeat forever
+  }
+  return {
+    systemMessage:
+      `AgentHub: ${rec.message}` +
+      (rec.log ? ` — details in ${rec.log}` : "") +
+      ". Run /agenthub-doctor for a diagnosis.",
+  };
+}
+
+function nodeTooOldMessage(version = process.versions.node) {
+  return (
+    `Node ${version} is too old — AgentHub needs Node >= ${MIN_NODE.join(".")} ` +
+    `(the server stores data through node:sqlite). Upgrade Node, then start a new session`
+  );
+}
+
 /* -------------------------------------------------------------------- lock */
 
 function lockPath(rt = runtimeDir()) {
@@ -422,13 +521,22 @@ function installDeps(rt = runtimeDir(), root = PLUGIN_ROOT) {
     if (fs.existsSync(src)) fs.copyFileSync(src, path.join(rt, f));
   }
   const npm = npmCommand(["install", "--omit=dev", "--ignore-scripts", "--no-audit", "--no-fund"]);
-  const res = spawnSync(npm.file, npm.args, {
-    cwd: rt,
-    stdio: "inherit",
-    env: npm.env,
-    shell: npm.shell,
-  });
-  return res.status === 0;
+  // npm's output goes to a log: the worker's own stdout is discarded, so
+  // inheriting it threw away the one explanation of a failed install.
+  const logPath = path.join(rt, "npm-install.log");
+  require("./log-rotate").rotateIfLarge(logPath);
+  const logFd = fs.openSync(logPath, "a");
+  try {
+    const res = spawnSync(npm.file, npm.args, {
+      cwd: rt,
+      stdio: ["ignore", logFd, logFd],
+      env: npm.env,
+      shell: npm.shell,
+    });
+    return res.status === 0;
+  } finally {
+    fs.closeSync(logFd);
+  }
 }
 
 /* ------------------------------------------------------------------ server */
@@ -671,11 +779,8 @@ function bootstrap(rt = runtimeDir(), { force = false } = {}) {
   }
 
   if (!nodeVersionOk()) {
-    log(
-      rt,
-      `Node ${process.versions.node} is too old — AgentHub needs Node >= ${MIN_NODE.join(".")} ` +
-        `(the server stores data through node:sqlite). Upgrade Node, then start a new session.`
-    );
+    log(rt, nodeTooOldMessage());
+    recordFailure(rt, "node-too-old", nodeTooOldMessage(), path.join(rt, "bootstrap.log"));
     return "node-too-old";
   }
 
@@ -695,7 +800,16 @@ function bootstrap(rt = runtimeDir(), { force = false } = {}) {
     if (force || state.depsHash !== wantDeps || !fs.existsSync(path.join(rt, "node_modules"))) {
       log(rt, "installing runtime dependencies (first run takes a few minutes)");
       if (!installDeps(rt)) {
-        log(rt, "npm install failed — see the output above; run /agenthub-doctor after fixing it");
+        log(
+          rt,
+          `npm install failed — see ${path.join(rt, "npm-install.log")}; run /agenthub-doctor after fixing it`
+        );
+        recordFailure(
+          rt,
+          "install-failed",
+          "installing the dashboard's runtime dependencies failed",
+          path.join(rt, "npm-install.log")
+        );
         return "install-failed";
       }
     }
@@ -713,17 +827,24 @@ function bootstrap(rt = runtimeDir(), { force = false } = {}) {
 
     // Built eagerly (not lazily behind /agenthub-open) so the dashboard, including
     // client-only routes like /run, works the instant `claude` is started —
-    // same trigger as the dependency install above: missing, or this plugin
-    // version has not built one yet.
+    // when missing, or when this plugin version has not built one yet
+    // (`clientBuild` is recorded only after a build that did not fail, so the
+    // next session retries a failed one instead of keeping the old UI for good).
     const { buildClient } = require("./plugin-open");
     const uiResult = buildClient({
       rt,
       root: PLUGIN_ROOT,
-      force: force || state.depsHash !== wantDeps,
+      force: force || state.clientBuild !== pluginVersion(),
       logPath: path.join(rt, "client-build.log"),
     });
     if (uiResult === "failed") {
       log(rt, "dashboard UI build failed — the API still works; run /agenthub-open to retry");
+      recordFailure(
+        rt,
+        "ui-build-failed",
+        "the dashboard UI build failed (the API still works; /agenthub-open retries it)",
+        path.join(rt, "client-build.log")
+      );
     } else if (uiResult === "built") {
       log(rt, "built the dashboard UI bundle");
     }
@@ -829,6 +950,7 @@ function bootstrap(rt = runtimeDir(), { force = false } = {}) {
       {
         pluginVersion: pluginVersion(),
         depsHash: wantDeps,
+        clientBuild: uiResult === "failed" ? state.clientBuild : pluginVersion(),
         pluginRoot: PLUGIN_ROOT,
         claudeHome: getClaudeHome(),
         updatedAt: new Date().toISOString(),
@@ -836,6 +958,8 @@ function bootstrap(rt = runtimeDir(), { force = false } = {}) {
       rt
     );
 
+    if (uiResult !== "failed") clearFailure(rt);
+
     // After the state is durable, so a prune that goes wrong cannot also cost
     // us the record of a bootstrap that went right.
     pruneRetiredPluginRoots(rt);
@@ -868,13 +992,26 @@ function bootstrap(rt = runtimeDir(), { force = false } = {}) {
  * milliseconds in the steady state, and otherwise hands off to a detached
  * worker so the session never waits on an install.
  */
-function main() {
-  if (notTheHost() || fastPathOk()) return;
-  const child = spawn(process.execPath, [__filename, "--worker"], {
-    detached: true,
-    stdio: "ignore",
-  });
-  child.unref();
+function main(rt = runtimeDir()) {
+  if (notTheHost()) return null;
+  // Checked here too, not only in the worker: it costs nothing, the worker
+  // could only log it, and this pass can tell the user right now. A retired
+  // build stays silent — its worker would do nothing either.
+  if (!nodeVersionOk()) {
+    if (supersededBy()) return null;
+    recordFailure(rt, "node-too-old", nodeTooOldMessage(), null);
+    return takeFailureReport(rt);
+  }
+  if (!fastPathOk(rt)) {
+    const child = spawn(process.execPath, [__filename, "--worker"], {
+      detached: true,
+      stdio: "ignore",
+    });
+    child.unref();
+  }
+  // A failure the previous worker recorded (it ran after that session's
+  // foreground pass had already returned). Pure fs; writes only to mark it shown.
+  return takeFailureReport(rt);
 }
 
 // Exported before the entry block below, not after it: `bootstrap()` requires
@@ -897,6 +1034,12 @@ module.exports = {
   depsHash,
   fastPathOk,
   nodeVersionOk,
+  failurePath,
+  readFailure,
+  recordFailure,
+  clearFailure,
+  takeFailureReport,
+  main,
   lockPath,
   lockIsStale,
   acquireLock,
@@ -934,7 +1077,10 @@ if (require.main === module) {
     }
   } else {
     try {
-      main();
+      const report = main();
+      // SessionStart reads a JSON object on stdout; `systemMessage` is shown
+      // to the user. Nothing at all is printed in the steady state.
+      if (report) process.stdout.write(JSON.stringify(report) + "\n");
     } catch {
       // A bootstrap failure must never break a Claude Code session.
     }

scripts/plugin-open.js

@@ -68,7 +68,19 @@ function buildClient({
   const stdio = logFd === null ? "inherit" : ["ignore", logFd, logFd];
   try {
     const src = path.join(rt, "client-src");
-    fs.rmSync(src, { recursive: true, force: true });
+    // The installed tree is kept across updates while the client's manifests
+    // are unchanged (`stamp`, written after a successful install): a release
+    // rarely touches them, and the clean `npm ci` below fetched all of the
+    // ~200 MB toolchain again on every update.
+    const stamp = path.join(src, "node_modules", ".agenthub-deps");
+    const want = clientDepsHash(root);
+    const reuse = readText(stamp) === want;
+    if (reuse) {
+      for (const e of fs.readdirSync(src))
+        if (e !== "node_modules") fs.rmSync(path.join(src, e), { recursive: true, force: true });
+    } else {
+      fs.rmSync(src, { recursive: true, force: true });
+    }
     fs.cpSync(path.join(root, "client"), src, {
       recursive: true,
       filter: (p) => !/[\\/](node_modules|dist)$/.test(p),
@@ -101,19 +113,25 @@ function buildClient({
     // esbuild is the only dependency with an install script, and it finds its
     // platform binary without it.
     const npmFlags = ["--no-audit", "--no-fund", "--include=dev", "--ignore-scripts"];
-    if (
-      !exec(NPM, ["ci", ...npmFlags], src, stdio) &&
-      !exec(NPM, ["install", ...npmFlags], src, stdio)
-    ) {
-      return "failed";
+    if (!reuse) {
+      if (
+        !exec(NPM, ["ci", ...npmFlags], src, stdio) &&
+        !exec(NPM, ["install", ...npmFlags], src, stdio)
+      ) {
+        return "failed";
+      }
+      fs.mkdirSync(path.dirname(stamp), { recursive: true });
+      fs.writeFileSync(stamp, want);
     }
-    // = client/package.json's "build" script (`tsc -b && vite build`); a test
-    // keeps the two in step.
+    // client/package.json's "build" script is `tsc -b && vite build`; only the
+    // vite half runs here. The type check emits nothing the bundle needs, and on
+    // a user's machine it cost ~25 s and ~900 MB of memory per update — and a
+    // type error in a release then kept the old UI everywhere (v2.7.0). The
+    // release itself still runs `tsc -b`. A test keeps the two in step.
     const bin = (...p) => path.join(src, "node_modules", ...p);
-    if (
-      !exec(NODE, [bin("typescript", "bin", "tsc"), "-b"], src, stdio) ||
-      !exec(NODE, [bin("vite", "bin", "vite.js"), "build"], src, stdio)
-    ) {
+    if (!exec(NODE, [bin("vite", "bin", "vite.js"), "build"], src, stdio)) {
+      // Maybe a damaged kept tree: the next build starts from a clean install.
+      fs.rmSync(stamp, { force: true });
       return "failed";
     }
 
@@ -125,6 +143,22 @@ function buildClient({
   }
 }
 
+function readText(file) {
+  try {
+    return fs.readFileSync(file, "utf8");
+  } catch {
+    return null;
+  }
+}
+
+/** The client's manifests, which decide whether its installed tree still fits. */
+function clientDepsHash(root) {
+  const h = require("crypto").createHash("sha256");
+  for (const f of ["package.json", "package-lock.json"])
+    h.update(readText(path.join(root, "client", f)) ?? `no ${f}`);
+  return h.digest("hex").slice(0, 16);
+}
+
 function dashboardUrl() {
   try {
     return `http://localhost:${resolveDashboardPort()}`;