feat(workflows): pipeline pass rates and skill/plugin usage
Restores the five analytics sections that turned out to be worth keeping (patterns, model delegation, error propagation, session complexity, compaction impact) and reworks what the page opens on. The lane panel no longer mirrors the Workspace's live lane strip. It now answers the question the strip cannot: how often each pipeline node actually passes. GET /api/lanes/pipeline-stats aggregates every stage record of every lane AND every archived feature — a lane's own `stages` only holds its current feature, so without the archive the history resets each time a lane starts new work. Evidence-backed passes are counted apart from bare claims, the same line the pipeline map draws between `done` and `passed-no-evidence`. Beside it, GET /api/workflows/skill-usage reports which skills and plugins the sessions actually reach for. Only PreToolUse rows are counted: every call also emits a PostToolUse, and counting both would double every number. A plugin's total includes calls to the MCP tools it ships, so one used purely through MCP does not read as unused. Still dropped from issue #6: the agent collaboration network and the subagent effectiveness scorecards.
This commit is contained in:
@@ -0,0 +1,162 @@
|
||||
/**
|
||||
* @file Tests for the two aggregate endpoints the Workflows page reads:
|
||||
* GET /api/lanes/pipeline-stats (how often each pipeline node was passed,
|
||||
* over lanes AND archived features) and GET /api/workflows/skill-usage
|
||||
* (per-skill and per-plugin call counts). Both are pure reporting endpoints,
|
||||
* so the risks are miscounting — double-counting a tool call that emits both a
|
||||
* Pre and a Post event, or losing an archived feature's history.
|
||||
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
|
||||
*/
|
||||
|
||||
const os = require("node:os");
|
||||
const path = require("node:path");
|
||||
const fs = require("node:fs");
|
||||
const http = require("node:http");
|
||||
|
||||
const SUITE_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "ccam-usage-api-"));
|
||||
process.env.DASHBOARD_DB_PATH = path.join(SUITE_ROOT, "dashboard.db");
|
||||
process.env.LANES_ROOT = path.join(SUITE_ROOT, "lanes");
|
||||
|
||||
const { describe, it, before, after } = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
|
||||
const { createApp } = require("../index");
|
||||
const lanesLib = require("../lib/lanes");
|
||||
const { db, stmts } = require("../db");
|
||||
|
||||
let server;
|
||||
let PORT;
|
||||
|
||||
before(async () => {
|
||||
const app = createApp();
|
||||
server = http.createServer(app);
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
PORT = server.address().port;
|
||||
});
|
||||
after(async () => {
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
fs.rmSync(SUITE_ROOT, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function get(urlPath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = http.request(
|
||||
{ method: "GET", hostname: "127.0.0.1", port: PORT, path: urlPath },
|
||||
(res) => {
|
||||
let raw = "";
|
||||
res.on("data", (c) => (raw += c));
|
||||
res.on("end", () => resolve({ status: res.statusCode, body: JSON.parse(raw) }));
|
||||
}
|
||||
);
|
||||
req.on("error", reject);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
let laneSeq = 0;
|
||||
function makeLane() {
|
||||
laneSeq += 1;
|
||||
const cwd = path.join(SUITE_ROOT, `lane-cwd-${laneSeq}`);
|
||||
fs.mkdirSync(cwd, { recursive: true });
|
||||
return lanesLib.createLane({ title: `lane ${laneSeq}`, cwd, kind: "managed" });
|
||||
}
|
||||
|
||||
function setStages(laneId, stages) {
|
||||
db.prepare("UPDATE lanes SET stages = ? WHERE id = ?").run(JSON.stringify(stages), laneId);
|
||||
}
|
||||
|
||||
function node(pipelineStats, pipelineId, nodeId) {
|
||||
const p = pipelineStats.pipelines.find((x) => x.id === pipelineId);
|
||||
return p && p.nodes.find((n) => n.id === nodeId);
|
||||
}
|
||||
|
||||
describe("GET /api/lanes/pipeline-stats", () => {
|
||||
it("counts a pass with evidence apart from a pass without one, and a fail as neither", async () => {
|
||||
const lane = makeLane();
|
||||
setStages(lane.id, {
|
||||
plan: { enteredAt: "2026-01-01T00:00:00Z", evidence: "plan.md", result: null },
|
||||
implement: { enteredAt: "2026-01-01T01:00:00Z", evidence: null, result: null },
|
||||
tests: { enteredAt: "2026-01-01T02:00:00Z", evidence: null, result: "fail" },
|
||||
});
|
||||
|
||||
const res = await get("/api/lanes/pipeline-stats");
|
||||
assert.equal(res.status, 200);
|
||||
|
||||
const plan = node(res.body, "default", "plan");
|
||||
assert.deepEqual(
|
||||
{ entered: plan.entered, passed: plan.passed, withEvidence: plan.withEvidence },
|
||||
{ entered: 1, passed: 1, withEvidence: 1 }
|
||||
);
|
||||
|
||||
const implement = node(res.body, "default", "implement");
|
||||
assert.deepEqual(
|
||||
{ passed: implement.passed, withEvidence: implement.withEvidence },
|
||||
{ passed: 1, withEvidence: 0 }
|
||||
);
|
||||
|
||||
const tests = node(res.body, "default", "tests");
|
||||
assert.deepEqual(
|
||||
{ entered: tests.entered, passed: tests.passed, failed: tests.failed },
|
||||
{ entered: 1, passed: 0, failed: 1 }
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves a node never reached at zero", async () => {
|
||||
const res = await get("/api/lanes/pipeline-stats");
|
||||
assert.equal(node(res.body, "default", "ship").entered, 0);
|
||||
});
|
||||
|
||||
it("keeps counting an archived feature's stages after the lane moves on", async () => {
|
||||
const lane = makeLane();
|
||||
db.prepare(
|
||||
`INSERT INTO lane_features (lane_id, slug, pipeline, stage, stages, archived_at)
|
||||
VALUES (?, 'old', 'default', 'done', ?, '2026-01-02T00:00:00Z')`
|
||||
).run(lane.id, JSON.stringify({ gate: { enteredAt: "x", evidence: "GO", result: null } }));
|
||||
|
||||
const res = await get("/api/lanes/pipeline-stats");
|
||||
const gate = node(res.body, "default", "gate");
|
||||
assert.equal(gate.passed, 1);
|
||||
assert.equal(gate.withEvidence, 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("GET /api/workflows/skill-usage", () => {
|
||||
before(() => {
|
||||
stmts.insertSession.run("usage-s1", "usage", "active", "/tmp/usage", null, null);
|
||||
const ev = (type, tool, data) =>
|
||||
stmts.insertEvent.run("usage-s1", null, type, tool, "", JSON.stringify(data));
|
||||
|
||||
// Two full calls of the same skill: each emits Pre + Post, so a naive
|
||||
// COUNT(*) would report four.
|
||||
for (let i = 0; i < 2; i += 1) {
|
||||
ev("PreToolUse", "Skill", { tool_input: { skill: "ccam:ship-feature" } });
|
||||
ev("PostToolUse", "Skill", { tool_input: { skill: "ccam:ship-feature" } });
|
||||
}
|
||||
ev("PreToolUse", "Skill", { tool_input: { skill: "daily-report" } });
|
||||
ev("PreToolUse", "mcp__plugin_ccam_ccam__dashboard_health_check", {});
|
||||
ev("PostToolUse", "mcp__plugin_ccam_ccam__dashboard_health_check", {});
|
||||
});
|
||||
|
||||
it("counts one call per invocation, not one per hook event", async () => {
|
||||
const res = await get("/api/workflows/skill-usage");
|
||||
assert.equal(res.status, 200);
|
||||
const ship = res.body.skills.find((s) => s.name === "ccam:ship-feature");
|
||||
assert.equal(ship.count, 2);
|
||||
assert.equal(ship.sessions, 1);
|
||||
});
|
||||
|
||||
it("attributes a prefixed skill to its plugin and leaves a bare one unattributed", async () => {
|
||||
const res = await get("/api/workflows/skill-usage");
|
||||
assert.equal(res.body.skills.find((s) => s.name === "ccam:ship-feature").plugin, "ccam");
|
||||
assert.equal(res.body.skills.find((s) => s.name === "daily-report").plugin, null);
|
||||
});
|
||||
|
||||
it("adds a plugin's own MCP tool calls to its totals", async () => {
|
||||
const res = await get("/api/workflows/skill-usage");
|
||||
const ccam = res.body.plugins.find((p) => p.name === "ccam");
|
||||
assert.deepEqual(
|
||||
{ skills: ccam.skills, skillCalls: ccam.skillCalls, mcpCalls: ccam.mcpCalls },
|
||||
{ skills: 1, skillCalls: 2, mcpCalls: 1 }
|
||||
);
|
||||
});
|
||||
});
|
||||
+74
-1
@@ -14,7 +14,13 @@ const os = require("node:os");
|
||||
const path = require("node:path");
|
||||
const { db } = require("../db");
|
||||
const lanesLib = require("../lib/lanes");
|
||||
const { listPipelines, getPipeline, nodeStates, progressPct } = require("../lib/pipelines");
|
||||
const {
|
||||
listPipelines,
|
||||
getPipeline,
|
||||
nodeStates,
|
||||
progressPct,
|
||||
stageRecords,
|
||||
} = require("../lib/pipelines");
|
||||
const laneFeatures = require("../lib/lane-features");
|
||||
const proofLib = require("../lib/proof");
|
||||
const { broadcast } = require("../websocket");
|
||||
@@ -128,6 +134,73 @@ router.get("/", (_req, res) => {
|
||||
// Registered before "/:id" so the literal path is not swallowed by the param.
|
||||
router.get("/pipelines", (_req, res) => res.json({ pipelines: listPipelines() }));
|
||||
|
||||
/**
|
||||
* How often each pipeline node has actually been passed, aggregated over every
|
||||
* lane AND every archived feature — an archived feature is a finished run of a
|
||||
* pipeline, so dropping it would throw away most of the history the moment a
|
||||
* lane starts its next feature.
|
||||
*
|
||||
* A node is counted `entered` when a stage record resolves onto it; `failed`
|
||||
* when that record says `result: "fail"`; `passed` otherwise; `withEvidence`
|
||||
* for the subset of passes that left an artifact behind. Evidence is tracked
|
||||
* separately rather than folded into the pass rate, because "claimed" and
|
||||
* "proved" are not the same fact — the same distinction the pipeline map draws
|
||||
* between `done` and `passed-no-evidence`. Also registered before "/:id".
|
||||
*/
|
||||
router.get("/pipeline-stats", (_req, res) => {
|
||||
const rows = [
|
||||
...db.prepare("SELECT pipeline, stages FROM lanes").all(),
|
||||
...db.prepare("SELECT pipeline, stages FROM lane_features").all(),
|
||||
];
|
||||
|
||||
const acc = new Map();
|
||||
for (const row of rows) {
|
||||
let stages;
|
||||
try {
|
||||
stages = JSON.parse(row.stages);
|
||||
} catch {
|
||||
continue; // a corrupt blob costs one row, never the whole report
|
||||
}
|
||||
const pipeline = getPipeline(row.pipeline);
|
||||
let entry = acc.get(pipeline.id);
|
||||
if (!entry) {
|
||||
entry = {
|
||||
id: pipeline.id,
|
||||
name: pipeline.name || pipeline.id,
|
||||
runs: 0,
|
||||
nodes: pipeline.nodes.map((n) => ({
|
||||
id: n.id,
|
||||
label: n.label,
|
||||
icon: n.icon,
|
||||
gate: n.gate,
|
||||
entered: 0,
|
||||
passed: 0,
|
||||
withEvidence: 0,
|
||||
failed: 0,
|
||||
})),
|
||||
};
|
||||
acc.set(pipeline.id, entry);
|
||||
}
|
||||
entry.runs += 1;
|
||||
const records = stageRecords(pipeline, stages);
|
||||
for (const [idx, rec] of records) {
|
||||
const node = entry.nodes[idx];
|
||||
if (!node) continue;
|
||||
node.entered += 1;
|
||||
if (rec && rec.result === "fail") node.failed += 1;
|
||||
else {
|
||||
node.passed += 1;
|
||||
if (rec && rec.evidence) node.withEvidence += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Busiest pipeline first: the one with the most recorded runs is the one the
|
||||
// reader most likely came to look at.
|
||||
const pipelines = [...acc.values()].sort((a, b) => b.runs - a.runs);
|
||||
res.json({ pipelines });
|
||||
});
|
||||
|
||||
/**
|
||||
* Local branches of a candidate source repo, for the "Add lane" picker: pick
|
||||
* a repo, then pick which branch to fork the new worktree from, instead of
|
||||
|
||||
@@ -39,6 +39,82 @@ router.get("/", (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* How much each skill and each plugin is actually used.
|
||||
*
|
||||
* Skill calls are read off `Skill` tool events, one row per invocation:
|
||||
* `PreToolUse` only, because every call also emits a `PostToolUse` and counting
|
||||
* both would double every number. The skill's own name carries its plugin as a
|
||||
* `plugin:skill` prefix; a bare name is a built-in or personal skill and is
|
||||
* grouped under `null`.
|
||||
*
|
||||
* A plugin's total also includes calls to the MCP tools it ships
|
||||
* (`mcp__plugin_<plugin>__<tool>`) — a plugin used entirely through its MCP
|
||||
* server would otherwise report zero usage.
|
||||
*/
|
||||
router.get("/skill-usage", (req, res) => {
|
||||
try {
|
||||
const sf = sessionIdFilter(req.query.status || null);
|
||||
|
||||
const skills = db
|
||||
.prepare(
|
||||
`SELECT json_extract(data, '$.tool_input.skill') AS name,
|
||||
COUNT(*) AS count,
|
||||
COUNT(DISTINCT session_id) AS sessions
|
||||
FROM events
|
||||
WHERE tool_name = 'Skill' AND event_type = 'PreToolUse'
|
||||
AND json_extract(data, '$.tool_input.skill') IS NOT NULL${sf.clause}
|
||||
GROUP BY name ORDER BY count DESC LIMIT 30`
|
||||
)
|
||||
.all(...sf.params)
|
||||
.map((r) => {
|
||||
const i = r.name.indexOf(":");
|
||||
return { ...r, plugin: i > 0 ? r.name.slice(0, i) : null };
|
||||
});
|
||||
|
||||
const mcpCalls = db
|
||||
.prepare(
|
||||
`SELECT tool_name, COUNT(*) AS count FROM events
|
||||
WHERE tool_name LIKE 'mcp__plugin\\_%' ESCAPE '\\'
|
||||
AND event_type = 'PreToolUse'${sf.clause}
|
||||
GROUP BY tool_name`
|
||||
)
|
||||
.all(...sf.params);
|
||||
|
||||
const byPlugin = new Map();
|
||||
const bump = (name, field, n) => {
|
||||
const e = byPlugin.get(name) || { name, skills: 0, skillCalls: 0, mcpCalls: 0 };
|
||||
e[field] += n;
|
||||
byPlugin.set(name, e);
|
||||
};
|
||||
for (const s of skills) {
|
||||
if (!s.plugin) continue;
|
||||
bump(s.plugin, "skills", 1);
|
||||
bump(s.plugin, "skillCalls", s.count);
|
||||
}
|
||||
for (const m of mcpCalls) {
|
||||
// mcp__plugin_<plugin>_<server>__<tool>. Both halves may contain
|
||||
// underscores, so the split is only unambiguous when the plugin names its
|
||||
// server after itself — the overwhelmingly common case (`ccam_ccam`,
|
||||
// `vercel_vercel`), which the dedupe below collapses back to one name.
|
||||
// ponytail: a plugin whose server has its own name reports under
|
||||
// "<plugin>_<server>"; read the real pairing off the MCP config if that
|
||||
// ever matters.
|
||||
const seg = m.tool_name.slice("mcp__plugin_".length).split("__")[0];
|
||||
const doubled = seg.match(/^(.+)_\1$/);
|
||||
const plugin = doubled ? doubled[1] : seg;
|
||||
if (plugin) bump(plugin, "mcpCalls", m.count);
|
||||
}
|
||||
const plugins = [...byPlugin.values()].sort(
|
||||
(a, b) => b.skillCalls + b.mcpCalls - (a.skillCalls + a.mcpCalls)
|
||||
);
|
||||
|
||||
res.json({ skills, plugins });
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: { message: err.message } });
|
||||
}
|
||||
});
|
||||
|
||||
// ── GET /session/:id — Single session drill-in ──
|
||||
router.get("/session/:id", (req, res) => {
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user