feat: Claude Code Monitor — lanes, pipelines and a merged workspace
Internal SmartGift build of a Claude Code monitoring dashboard. Lanes: a durable unit of parallel agent work, one per working directory, tracked across session restarts. Managed lanes are git worktrees the dashboard provisions and can reset or remove behind a three-check destroy guard and a counted preflight; adopted lanes are directories you already own and are never destroyable. Pipelines: a lane moves through pipeline stages. A stage the agent declares with evidence renders green; a stage inferred from the tool-event stream renders dashed amber and never counts as done. Detection is forward-only within a 30-minute window, and never writes the declared stage. Workspace: one page at /run with a lane grid, the selected lane's pipeline, and a full Claude console behind a disclosure.
This commit is contained in:
@@ -0,0 +1,421 @@
|
||||
/**
|
||||
* @file Express router for the Import History feature.
|
||||
*
|
||||
* Three entry points, all of which funnel into the exact same parser +
|
||||
* `importSession` pipeline the server uses for live ingestion — guaranteeing
|
||||
* that imported tokens, per-model breakdowns, cost calculations, compactions,
|
||||
* subagents, tool events, API errors, and turn durations line up bit-for-bit
|
||||
* with sessions captured in real time.
|
||||
*
|
||||
* GET /api/import/guide — OS-aware instructions + default paths
|
||||
* POST /api/import/rescan — re-scan the default ~/.claude/projects dir
|
||||
* POST /api/import/scan-path — scan an arbitrary absolute directory path
|
||||
* POST /api/import/upload — multipart: JSONLs and/or archives
|
||||
*
|
||||
* Progress is broadcast over the existing websocket as `import.progress`.
|
||||
*
|
||||
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
|
||||
*/
|
||||
|
||||
const { Router } = require("express");
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const os = require("os");
|
||||
|
||||
const { broadcast } = require("../websocket");
|
||||
const {
|
||||
importAllSessions,
|
||||
importFromDirectory,
|
||||
collectJsonlFiles,
|
||||
} = require("../../scripts/import-history");
|
||||
const {
|
||||
mkTempDir,
|
||||
rmTempDir,
|
||||
extractInto,
|
||||
detectKind,
|
||||
ExtractionLimitError,
|
||||
} = require("../lib/archive");
|
||||
|
||||
const router = Router();
|
||||
|
||||
const { getClaudeHome, getProjectsDir } = require("../lib/claude-home");
|
||||
|
||||
// Upload limits — deliberately generous because transcripts can be large.
|
||||
// Configurable at runtime via env for deployments that need tighter bounds.
|
||||
const MAX_UPLOAD_BYTES = parseInt(
|
||||
process.env.CCAM_IMPORT_MAX_BYTES || String(1024 * 1024 * 1024), // 1 GB default
|
||||
10
|
||||
);
|
||||
const MAX_UPLOAD_FILES = parseInt(process.env.CCAM_IMPORT_MAX_FILES || "2000", 10);
|
||||
|
||||
/**
|
||||
* Lazily build a multer upload middleware. Kept lazy so the server still
|
||||
* boots if `multer` isn't installed yet — only /upload fails in that case.
|
||||
*
|
||||
* Each request gets its own staging directory created on the `req` object
|
||||
* during the first call to `destination`. Multer invokes `destination` once
|
||||
* per uploaded file, all within the same request, so a sentinel on `req`
|
||||
* avoids creating multiple dirs per request while guaranteeing isolation
|
||||
* across concurrent requests.
|
||||
*/
|
||||
function getUploader() {
|
||||
let multer;
|
||||
try {
|
||||
multer = require("multer");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const storage = multer.diskStorage({
|
||||
destination: (req, _file, cb) => {
|
||||
if (!req._ccamUploadDir) req._ccamUploadDir = mkTempDir("ccam-upload-");
|
||||
cb(null, req._ccamUploadDir);
|
||||
},
|
||||
filename: (_req, file, cb) => {
|
||||
// Preserve the original name for kind-detection later, but prefix with
|
||||
// a random token so collisions between two uploads with the same name
|
||||
// don't clobber each other.
|
||||
const rand = require("crypto").randomBytes(4).toString("hex");
|
||||
cb(null, `${rand}__${file.originalname}`);
|
||||
},
|
||||
});
|
||||
return multer({
|
||||
storage,
|
||||
limits: {
|
||||
files: MAX_UPLOAD_FILES,
|
||||
fileSize: MAX_UPLOAD_BYTES,
|
||||
fields: 32,
|
||||
},
|
||||
fileFilter: (req, file, cb) => {
|
||||
const kind = detectKind(file.originalname);
|
||||
if (kind === "unknown") {
|
||||
// Track rejected filenames on the request so we can surface the count
|
||||
// in the response — users wonder why their upload "partially worked".
|
||||
if (!req._ccamRejected) req._ccamRejected = [];
|
||||
req._ccamRejected.push(file.originalname);
|
||||
cb(null, false);
|
||||
} else {
|
||||
cb(null, true);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Throttle progress broadcasts so we don't flood the websocket on large imports.
|
||||
*/
|
||||
function makeProgressBroadcaster(importId) {
|
||||
let lastSent = 0;
|
||||
return (progress) => {
|
||||
const now = Date.now();
|
||||
if (progress.phase === "complete" || now - lastSent > 150) {
|
||||
lastSent = now;
|
||||
broadcast("import.progress", { importId, ...progress });
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function countsSummary(counters) {
|
||||
return {
|
||||
imported: counters.imported || 0,
|
||||
skipped: counters.skipped || 0,
|
||||
backfilled: counters.backfilled || 0,
|
||||
errors: counters.errors || 0,
|
||||
sessions_seen: counters.sessionsSeen || 0,
|
||||
files_scanned: counters.filesScanned || 0,
|
||||
};
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// GET /api/import/guide — step-by-step instructions the UI renders verbatim.
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
router.get("/guide", (_req, res) => {
|
||||
const platform = process.platform;
|
||||
const claudeHome = getClaudeHome();
|
||||
const claudeHomeDisplay = claudeHome.replace(os.homedir(), "~");
|
||||
const projectsDisplay = path.join(claudeHomeDisplay, "projects");
|
||||
const defaults = {
|
||||
darwin: projectsDisplay,
|
||||
linux: projectsDisplay,
|
||||
win32: projectsDisplay.replace(/\//g, "\\"),
|
||||
};
|
||||
const archiveBase = claudeHomeDisplay;
|
||||
const archiveCmd = {
|
||||
darwin: `tar -czf claude-history.tar.gz -C ${archiveBase} projects`,
|
||||
linux: `tar -czf claude-history.tar.gz -C ${archiveBase} projects`,
|
||||
win32: `tar -czf claude-history.tar.gz -C "${projectsDisplay.replace(/\//g, "\\")}" projects`,
|
||||
};
|
||||
const exists = fs.existsSync(getProjectsDir());
|
||||
let projectCount = 0;
|
||||
let fileCount = 0;
|
||||
if (exists) {
|
||||
try {
|
||||
const dirs = fs
|
||||
.readdirSync(getProjectsDir(), { withFileTypes: true })
|
||||
.filter((d) => d.isDirectory());
|
||||
projectCount = dirs.length;
|
||||
for (const d of dirs) {
|
||||
try {
|
||||
fileCount += fs
|
||||
.readdirSync(path.join(getProjectsDir(), d.name))
|
||||
.filter((f) => f.endsWith(".jsonl")).length;
|
||||
} catch {
|
||||
/* non-fatal */
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* non-fatal */
|
||||
}
|
||||
}
|
||||
|
||||
res.json({
|
||||
platform,
|
||||
default_projects_dir: getProjectsDir(),
|
||||
default_projects_dir_display: defaults[platform] || getProjectsDir(),
|
||||
default_projects_dir_exists: exists,
|
||||
default_projects_dir_stats: { projects: projectCount, jsonl_files: fileCount },
|
||||
archive_command: archiveCmd[platform] || archiveCmd.linux,
|
||||
supported_extensions: [".jsonl", ".meta.json", ".zip", ".tar", ".tar.gz", ".tgz", ".gz"],
|
||||
max_upload_bytes: MAX_UPLOAD_BYTES,
|
||||
max_upload_files: MAX_UPLOAD_FILES,
|
||||
steps: [
|
||||
{
|
||||
id: "locate",
|
||||
title: "Locate your Claude Code history",
|
||||
body: `Claude Code stores every session as a JSONL transcript under ${defaults[platform] || defaults.linux}. Each subdirectory is named after the working directory where the session started (with slashes replaced by dashes).`,
|
||||
},
|
||||
{
|
||||
id: "archive",
|
||||
title: "Bundle it for transfer (optional)",
|
||||
body: `If you're importing from another machine, archive the whole projects folder first:\n\n ${archiveCmd[platform] || archiveCmd.linux}\n\nMove claude-history.tar.gz to this machine however you like (AirDrop, scp, USB, cloud storage).`,
|
||||
},
|
||||
{
|
||||
id: "choose",
|
||||
title: "Pick an import mode",
|
||||
body: "Rescan default: re-read ~/.claude/projects on this machine and import anything new. From folder: point the dashboard at any directory you've extracted history into. Upload: drag-drop JSONL files or an archive directly into the browser.",
|
||||
},
|
||||
{
|
||||
id: "verify",
|
||||
title: "Verify tokens and cost",
|
||||
body: "Imports are idempotent: re-running is always safe. Token counts are deduplicated per session ID, with compaction baselines preserved so cost never double-counts. After import, open Analytics → Cost to confirm the breakdown.",
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// POST /api/import/rescan — default ~/.claude/projects directory.
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
router.post("/rescan", async (_req, res) => {
|
||||
const importId = `rescan-${Date.now()}`;
|
||||
try {
|
||||
broadcast("import.progress", { importId, phase: "start", source: "default" });
|
||||
const dbModule = require("../db");
|
||||
const result = await importAllSessions(dbModule);
|
||||
broadcast("import.progress", {
|
||||
importId,
|
||||
phase: "complete",
|
||||
source: "default",
|
||||
counters: result,
|
||||
});
|
||||
res.json({ ok: true, source: "default", ...result });
|
||||
} catch (err) {
|
||||
broadcast("import.progress", { importId, phase: "error", error: err.message });
|
||||
res.status(500).json({ error: { code: "IMPORT_FAILED", message: err.message } });
|
||||
}
|
||||
});
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// POST /api/import/scan-path — arbitrary absolute directory.
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
router.post("/scan-path", async (req, res) => {
|
||||
const importId = `scan-${Date.now()}`;
|
||||
const rawPath = (req.body && req.body.path) || "";
|
||||
if (typeof rawPath !== "string" || !rawPath.trim()) {
|
||||
return res
|
||||
.status(400)
|
||||
.json({ error: { code: "INVALID_INPUT", message: "`path` is required" } });
|
||||
}
|
||||
|
||||
// Expand ~ to the user's home directory for convenience.
|
||||
const expanded = rawPath.startsWith("~") ? path.join(os.homedir(), rawPath.slice(1)) : rawPath;
|
||||
if (!path.isAbsolute(expanded)) {
|
||||
return res.status(400).json({
|
||||
error: { code: "INVALID_INPUT", message: "`path` must be an absolute path" },
|
||||
});
|
||||
}
|
||||
|
||||
let stat;
|
||||
try {
|
||||
stat = fs.statSync(expanded);
|
||||
} catch (err) {
|
||||
return res.status(400).json({
|
||||
error: { code: "PATH_NOT_FOUND", message: `Path does not exist: ${expanded}` },
|
||||
});
|
||||
}
|
||||
if (!stat.isDirectory()) {
|
||||
return res.status(400).json({
|
||||
error: { code: "NOT_A_DIRECTORY", message: `Path is not a directory: ${expanded}` },
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const onProgress = makeProgressBroadcaster(importId);
|
||||
broadcast("import.progress", { importId, phase: "start", source: "path", path: expanded });
|
||||
const dbModule = require("../db");
|
||||
const counters = await importFromDirectory(dbModule, expanded, { onProgress });
|
||||
const summary = countsSummary(counters);
|
||||
broadcast("import.progress", {
|
||||
importId,
|
||||
phase: "complete",
|
||||
source: "path",
|
||||
counters: summary,
|
||||
});
|
||||
res.json({ ok: true, source: "path", path: expanded, ...summary });
|
||||
} catch (err) {
|
||||
broadcast("import.progress", { importId, phase: "error", error: err.message });
|
||||
res.status(500).json({ error: { code: "IMPORT_FAILED", message: err.message } });
|
||||
}
|
||||
});
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// POST /api/import/upload — multipart: JSONL files and/or archives.
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
const uploader = getUploader();
|
||||
const uploadMiddleware = uploader
|
||||
? uploader.array("files", MAX_UPLOAD_FILES)
|
||||
: (_req, _res, next) => next();
|
||||
|
||||
router.post("/upload", uploadMiddleware, async (req, res) => {
|
||||
const importId = `upload-${Date.now()}`;
|
||||
if (!uploader) {
|
||||
return res.status(500).json({
|
||||
error: {
|
||||
code: "UPLOADER_UNAVAILABLE",
|
||||
message: "File upload requires `multer`. Run `npm install` to pick up new deps.",
|
||||
},
|
||||
});
|
||||
}
|
||||
const files = Array.isArray(req.files) ? req.files : [];
|
||||
const rejectedNames = Array.isArray(req._ccamRejected) ? req._ccamRejected : [];
|
||||
const reqUploadDir = req._ccamUploadDir || null;
|
||||
|
||||
if (files.length === 0) {
|
||||
// Clean up the upload dir if multer created one before rejecting all files.
|
||||
if (reqUploadDir) rmTempDir(reqUploadDir);
|
||||
return res.status(400).json({
|
||||
error: {
|
||||
code: "NO_FILES",
|
||||
message:
|
||||
rejectedNames.length > 0
|
||||
? `No supported files in upload. ${rejectedNames.length} file(s) rejected (unsupported extension).`
|
||||
: "No files received",
|
||||
},
|
||||
rejected_files: rejectedNames,
|
||||
});
|
||||
}
|
||||
|
||||
const workDir = mkTempDir("ccam-import-work-");
|
||||
let extractedCount = 0;
|
||||
let skippedEntries = 0;
|
||||
|
||||
try {
|
||||
broadcast("import.progress", {
|
||||
importId,
|
||||
phase: "extract",
|
||||
source: "upload",
|
||||
total: files.length,
|
||||
processed: 0,
|
||||
});
|
||||
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const f = files[i];
|
||||
try {
|
||||
const result = await extractInto(f.path, workDir, f.originalname);
|
||||
extractedCount += result.extracted;
|
||||
skippedEntries += result.skipped;
|
||||
} catch (err) {
|
||||
if (err instanceof ExtractionLimitError) {
|
||||
broadcast("import.progress", {
|
||||
importId,
|
||||
phase: "error",
|
||||
error: err.message,
|
||||
});
|
||||
return res.status(413).json({
|
||||
error: { code: err.code, message: err.message },
|
||||
offending_file: f.originalname,
|
||||
});
|
||||
}
|
||||
skippedEntries += 1;
|
||||
broadcast("import.progress", {
|
||||
importId,
|
||||
phase: "extract_error",
|
||||
current: f.originalname,
|
||||
error: err.message,
|
||||
});
|
||||
}
|
||||
broadcast("import.progress", {
|
||||
importId,
|
||||
phase: "extract",
|
||||
source: "upload",
|
||||
processed: i + 1,
|
||||
total: files.length,
|
||||
current: f.originalname,
|
||||
});
|
||||
}
|
||||
|
||||
// Even if extraction yielded zero files, the user may have uploaded a single
|
||||
// JSONL that was copied directly — `collectJsonlFiles` will find it.
|
||||
const jsonlPresent = collectJsonlFiles(workDir).length;
|
||||
if (jsonlPresent === 0) {
|
||||
return res.status(400).json({
|
||||
error: {
|
||||
code: "NO_JSONL",
|
||||
message:
|
||||
"No .jsonl files were found in the uploaded content. Supported inputs: .jsonl, .meta.json, .zip, .tar, .tar.gz, .tgz, .gz.",
|
||||
},
|
||||
extracted: extractedCount,
|
||||
skipped_entries: skippedEntries,
|
||||
});
|
||||
}
|
||||
|
||||
const onProgress = makeProgressBroadcaster(importId);
|
||||
const dbModule = require("../db");
|
||||
const counters = await importFromDirectory(dbModule, workDir, { onProgress });
|
||||
const summary = countsSummary(counters);
|
||||
|
||||
broadcast("import.progress", {
|
||||
importId,
|
||||
phase: "complete",
|
||||
source: "upload",
|
||||
counters: summary,
|
||||
});
|
||||
|
||||
res.json({
|
||||
ok: true,
|
||||
source: "upload",
|
||||
files_received: files.length,
|
||||
rejected_files: rejectedNames,
|
||||
entries_extracted: extractedCount,
|
||||
entries_skipped: skippedEntries,
|
||||
...summary,
|
||||
});
|
||||
} catch (err) {
|
||||
broadcast("import.progress", { importId, phase: "error", error: err.message });
|
||||
res.status(500).json({ error: { code: "IMPORT_FAILED", message: err.message } });
|
||||
} finally {
|
||||
// Always reclaim disk: the per-request staging dir, the extraction work
|
||||
// dir, and any loose multer files (usually subsumed by the staging dir,
|
||||
// but we unlink explicitly in case multer kept them elsewhere).
|
||||
rmTempDir(workDir);
|
||||
for (const f of files) {
|
||||
try {
|
||||
fs.unlinkSync(f.path);
|
||||
} catch {
|
||||
/* non-fatal */
|
||||
}
|
||||
}
|
||||
if (reqUploadDir) rmTempDir(reqUploadDir);
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
Reference in New Issue
Block a user