feat(health): System Health in three groups, a score that sees lag; admin user detail

Dashboard → System Health is grouped into AgentHub, Machine and Agent
activity. The machine (GET /api/host, every signed-in user) is drawn by
components/health/HostHealth.tsx: host card beside the CPU/memory history,
then CPU (threads as compact cells + temperatures/GPUs), disks & network,
busiest processes, and per-user tiles. Concurrency bars each show their own
agent type on hover.

The Health Score (client/src/lib/healthScore.ts) is now 100 minus what each
check costs, between an amber and a red threshold: event-loop p99
(server/lib/loop-delay.js, shared with /api/metrics), Linux PSI pressure for
CPU, memory and disk I/O (/api/host `pressure`; load per core without PSI),
RAM, the database's disk, V8 heap, hooks silent while agents work, and agent
errors in the last 24 h. All-time success rate, API errors and the transcript
cache hit rate no longer count. /api/settings/info adds server.event_loop and
activity {last_event_at, working_agents, agents_24h}.

Administration: a user row opens its detail in place (GET
/api/users/:id/details — spend, account pool share, live panes, workspaces,
newest sessions); the server panel is gone, member tmux folded into the table;
the Sessions list shows each row's owner in server mode.
This commit is contained in:
2026-10-27 16:39:38 +07:00
parent df15789982
commit 1c6dfea5b5
49 changed files with 5154 additions and 1971 deletions
@@ -0,0 +1,153 @@
/**
* @file healthScore.test.ts
* @description Tests for lib/healthScore.ts — the System Health score: a calm
* machine scores 100, lag (event loop, CPU / memory / I/O pressure) pulls it
* down hard, a check without data costs nothing, CPU falls back to load per
* core without PSI, the data disk is the one holding the database, and hooks
* count only while an agent works.
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
*/
import { describe, expect, it } from "vitest";
import type { HostStats } from "../api";
import { healthChecks, healthScore, scoreLevel, type HealthInput } from "../healthScore";
const GB = 1024 ** 3;
const NOW = Date.parse("2026-10-27T10:00:00Z");
const host = (over: Partial<HostStats> = {}): HostStats => ({
at: "2026-10-27T10:00:00Z",
hostname: "box",
os: { platform: "linux", release: "7", distro: null, arch: "x64" },
uptime: 1000,
cpu: { model: null, cores: 4, load: [0.5, 0.5, 0.5] },
memory: { total: 16 * GB, available: 12 * GB, used: 4 * GB, swapTotal: 0, swapUsed: 0 },
disks: [
{ device: "/dev/a", mount: "/", type: "ext4", kind: "SSD", total: 100, used: 10, free: 90 },
{ device: "/dev/b", mount: "/data", type: "ext4", kind: "HDD", total: 100, used: 99, free: 1 },
],
homesMount: null,
gpus: [],
usage: null,
pressure: {
cpu: { some10: 1, some60: 1, full10: 0 },
memory: { some10: 0, some60: 0, full10: 0 },
io: { some10: 0, some60: 0, full10: 0 },
},
...over,
});
const calm = (over: Partial<HealthInput> = {}): HealthInput => ({
eventLoop: { p50: 10, p99: 20, max: 40 },
heap: { heapUsed: 100, heapTotal: 200, heapLimit: 1000 },
hooksInstalled: true,
activity: {
last_event_at: "2026-10-27T09:59:00Z",
working_agents: 1,
agents_24h: { completed: 50, error: 0 },
},
dbPath: "/home/me/.claude/agenthub/dashboard.db",
host: host(),
now: NOW,
...over,
});
const byKey = (input: HealthInput) => new Map(healthChecks(input).map((c) => [c.key, c]));
describe("healthScore", () => {
it("scores a calm machine 100 with every check ok", () => {
const checks = healthChecks(calm());
expect(healthScore(checks)).toBe(100);
expect(checks.every((c) => c.level === "ok")).toBe(true);
expect(scoreLevel(100)).toBe("ok");
});
it("drops the score when the machine lags", () => {
const lagging = calm({
eventLoop: { p50: 50, p99: 1200, max: 2000 },
host: host({
pressure: {
cpu: { some10: 70, some60: 50, full10: 0 },
memory: { some10: 0, some60: 0, full10: 0 },
io: { some10: 60, some60: 40, full10: 30 },
},
}),
});
const c = byKey(lagging);
expect(c.get("loop")?.level).toBe("bad");
expect(c.get("cpu")?.level).toBe("bad");
expect(c.get("io")?.level).toBe("bad");
const score = healthScore([...c.values()]);
expect(score).toBe(100 - 25 - 20 - 20);
expect(scoreLevel(score)).toBe("bad");
});
it("charges part of the points between amber and red", () => {
const c = byKey(calm({ eventLoop: { p50: 10, p99: 550, max: 600 } })).get("loop");
expect(c?.level).toBe("warn");
expect(c?.penalty).toBe(12.5);
});
it("costs nothing for what was not measured, and estimates CPU from load without PSI", () => {
const c = byKey(
calm({
eventLoop: undefined,
host: host({ pressure: null, cpu: { model: null, cores: 2, load: [5, 1, 1] } }),
})
);
expect(c.get("loop")?.level).toBe("na");
expect(c.get("memory")?.level).toBe("na");
expect(c.get("cpu")?.detail?.source).toBe("load");
expect(c.get("cpu")?.value).toBe(250);
expect(c.get("cpu")?.level).toBe("bad");
});
it("reads the disk that holds the database", () => {
const c = byKey(calm({ dbPath: "/data/agenthub/dashboard.db" })).get("disk");
expect(c?.detail?.mount).toBe("/data");
expect(c?.level).toBe("bad");
expect(byKey(calm({ dbPath: null })).get("disk")?.detail?.mount).toBe("/");
});
it("counts silent hooks only while an agent works; missing hooks are red", () => {
const silent = calm({
activity: {
last_event_at: "2026-10-27T09:20:00Z",
working_agents: 2,
agents_24h: { completed: 0, error: 0 },
},
});
expect(byKey(silent).get("hooks")?.level).toBe("bad");
const idle = calm({
activity: {
last_event_at: "2026-10-27T05:00:00Z",
working_agents: 0,
agents_24h: { completed: 0, error: 0 },
},
});
expect(byKey(idle).get("hooks")).toMatchObject({ level: "ok", penalty: 0 });
expect(byKey(calm({ hooksInstalled: false })).get("hooks")).toMatchObject({
level: "bad",
penalty: 30,
});
});
it("rates agent errors only with enough finished agents", () => {
const few = calm({
activity: {
last_event_at: null,
working_agents: 0,
agents_24h: { completed: 1, error: 2 },
},
});
expect(byKey(few).get("agents")?.level).toBe("na");
const many = calm({
activity: {
last_event_at: null,
working_agents: 0,
agents_24h: { completed: 6, error: 4 },
},
});
expect(byKey(many).get("agents")).toMatchObject({ level: "bad", value: 40 });
});
});
+159 -8
View File
@@ -546,6 +546,83 @@ export type ServerUserUsage = AuthUser & {
logins?: number;
lastLoginAt?: string | null;
};
/** A priced slice of a user's spend (`GET /api/users/:id/details`). */
export type UsageSlice = { key: string; cost: number; tokens: number; sessions: number };
/** One pool account as one user's detail sees it: the account's totals and their share. */
export type UserPoolAccount = {
name: string;
kind: string;
email: string | null;
expired: boolean;
/** Their group/pin lets them run on it (false = they ran on it before). */
allowed: boolean;
pinned: boolean;
quota: PoolAccount["quota"] | CodexPoolAccount["quota"];
quotaPercent: number | null;
full: boolean;
livePanes: number;
spend5h: number;
spend7d: number;
userSpend5h: number;
userSpend7d: number;
userTokens5h: number;
userTokens7d: number;
userRuns7d: number;
userLivePanes: number;
userLastAt: string | null;
};
/** `GET /api/users/:id/details` — the Administration page's user detail. */
export type UserDetails = {
user: AuthUser & { disabled: boolean; logins: number; lastLoginAt: string | null };
sessions: { total: number; active: number; firstAt: string | null; lastAt: string | null };
usage: {
today: { cost: number; tokens: number };
last7d: { cost: number; tokens: number };
last30d: { cost: number; tokens: number };
/** 30 local days, the oldest first. */
daily: { date: string; cost: number; tokens: number }[];
byTool: UsageSlice[];
byModel: UsageSlice[];
};
workspaces: {
id: number;
title: string;
cwd: string;
tool: string;
kind: string;
branch: string | null;
status: string;
createdAt: string;
sessions: number;
lastActiveAt: string | null;
livePanes: number;
}[];
panes: {
id: string;
laneId: number;
laneTitle: string;
laneCwd: string;
tool: string;
model: string | null;
account: string | null;
proxy: string | null;
sessionId: string | null;
startedAt: string;
status: string;
}[];
recentSessions: (Session & { tokens?: number })[];
pool: Partial<
Record<
PoolTool,
{
pin: string | null;
group: { id: number; name: string } | null;
accounts: UserPoolAccount[];
}
>
>;
audit: AuditEntry[];
};
/** One member's own tmux server (`GET /api/users/tmux`). */
export interface MemberTmuxRow {
id: number;
@@ -578,6 +655,8 @@ export interface HostUsageGroup {
kind: "user" | "host" | "system";
userId?: number;
username?: string;
/** The user's display name, null when none is set (absent from an older server). */
displayName?: string | null;
/** % of the whole machine's CPU (100 = every core busy) over the sample. */
cpu: number;
/** Resident memory, bytes (shared pages count in every process). */
@@ -587,13 +666,31 @@ export interface HostUsageGroup {
/** Busiest processes, by the kernel's short name only. */
top: { pid: number; name: string; cpu: number; memory: number }[];
}
/** One `/proc/pressure/<resource>` reading, in % of the window. */
export interface HostPressure {
some10: number;
some60: number;
full10: number;
}
/** `GET /api/host` — hardware, load, disks and per-user usage. */
export interface HostStats {
at: string;
hostname: string;
os: { platform: string; release: string; distro: string | null; arch: string };
uptime: number;
cpu: { model: string | null; cores: number; load: number[] };
/** When the machine booted (ISO); absent from an older server. */
bootTime?: string;
cpu: {
model: string | null;
/** Logical CPUs (threads). */
cores: number;
load: number[];
/** Physical cores from /proc/cpuinfo; null when it cannot tell. */
physicalCores?: number | null;
/** Busy % of each logical CPU over the sample; [] off Linux. */
perCore?: number[];
};
memory: { total: number; available: number; used: number; swapTotal: number; swapUsed: number };
disks: {
device: string;
@@ -609,6 +706,46 @@ export interface HostStats {
gpus: { name: string; utilization: number; memoryUsed: number; memoryTotal: number }[];
/** null off Linux. */
usage: { cpu: number; sampleMs: number; groups: HostUsageGroup[] } | null;
/** Bytes/s per network interface over the sample (`lo` left out). */
network?: { name: string; rx: number; tx: number }[];
/** Bytes/s per whole disk over the sample. */
diskIo?: { name: string; read: number; write: number }[];
/** `/sys/class/hwmon` sensors; [] when none. */
temperatures?: { chip: string; label: string; celsius: number }[];
/** Linux PSI (`/proc/pressure/*`): % of the last 10 / 60 s tasks stalled on
* each resource; null without PSI (or off Linux). */
pressure?: {
cpu: HostPressure | null;
memory: HostPressure | null;
io: HostPressure | null;
} | null;
/** The ten busiest processes machine-wide; `group` is a `HostUsageGroup` key. */
topProcesses?: { pid: number; name: string; cpu: number; memory: number; group: string }[];
/** The dashboard's own process. */
process?: { pid: number; node: string; uptime: number; rss: number; heapUsed: number };
}
/** One point of `GET /api/host/history`. */
export interface HostHistoryPoint {
at: string;
/** Machine CPU %, null when it could not be sampled. */
cpu: number | null;
/** Memory used, % of total. */
memory: number | null;
memoryUsed: number;
swapUsed: number;
load1: number | null;
/** Per user id: CPU % and resident bytes (users with any use only). */
users: Record<string, { cpu: number; memory: number }>;
}
export type HostHistoryRange = "1h" | "6h" | "24h";
/** `GET /api/host/history` — kept in memory since the server started. */
export interface HostHistory {
range: HostHistoryRange;
intervalMs: number;
/** False when this server keeps no history (local mode, not Linux). */
running: boolean;
startedAt: string | null;
points: HostHistoryPoint[];
}
/** One user's home folder on disk (`GET /api/users/storage`); null = not measured. */
@@ -1090,9 +1227,11 @@ export const api = {
/** DELETE /api/me/avatar — back to the initials. */
removeAvatar: () => request<MyAccount>("/me/avatar", { method: "DELETE" }),
},
/** GET /api/host — the Server tab: hardware and each user's CPU/memory share (staff). */
/** GET /api/host — System Health's machine cards and the Administration users table's CPU/RAM (local mode, or staff). */
host: {
stats: () => request<HostStats>("/host"),
history: (range: HostHistoryRange = "1h") =>
request<HostHistory>(`/host/history?range=${range}`),
},
// ───────────────────────── Server mode staff views (routes/users.js) ─────────────────────────
users: {
@@ -1102,6 +1241,9 @@ export const api = {
/** GET /api/users — every account with its recent usage (admin/analyst). */
list: () =>
request<{ users: ServerUserUsage[] }>(`/users?tz_offset=${new Date().getTimezoneOffset()}`),
/** GET /api/users/:id/details — one account in full (admin/analyst). */
details: (id: number) =>
request<UserDetails>(`/users/${id}/details?tz_offset=${new Date().getTimezoneOffset()}`),
/** GET /api/users/audit — staff reads of members' sessions and admins'
* account changes, newest first. */
audit: () => request<{ entries: AuditEntry[] }>("/users/audit"),
@@ -2010,10 +2152,13 @@ export const api = {
purged_agents: number;
}>("/settings/cleanup", { method: "POST", body: JSON.stringify(params) }),
gc: (body: { dryRun?: boolean } = {}) =>
request<{ reaped: number[]; capped: { path: string; sizeBefore: number }[] }>("/workspaces/gc", {
method: "POST",
body: JSON.stringify(body),
}),
request<{ reaped: number[]; capped: { path: string; sizeBefore: number }[] }>(
"/workspaces/gc",
{
method: "POST",
body: JSON.stringify(body),
}
),
},
// ─────────────────────────────── Workflows API ──────────────────────────────
@@ -4288,7 +4433,10 @@ export const api = {
/** GET /api/lanes/:id/memory — CLAUDE.md/AGENTS.md and Claude's auto memory, read-only. */
memory: (id: number) => request<{ files: LaneMemoryFile[] }>(`/workspaces/${id}/memory`),
update: (id: number, patch: Partial<Lane>) =>
request<{ lane: Lane }>(`/workspaces/${id}`, { method: "PATCH", body: JSON.stringify(patch) }),
request<{ lane: Lane }>(`/workspaces/${id}`, {
method: "PATCH",
body: JSON.stringify(patch),
}),
/**
* POST /api/lanes/:id/stage — advance the pipeline stage.
* @param id The lane id.
@@ -4299,7 +4447,10 @@ export const api = {
id: number,
body: { stage: string; status?: string; evidence?: string; note?: string; result?: string }
) =>
request<{ lane: Lane }>(`/workspaces/${id}/stage`, { method: "POST", body: JSON.stringify(body) }),
request<{ lane: Lane }>(`/workspaces/${id}/stage`, {
method: "POST",
body: JSON.stringify(body),
}),
/**
* GET /api/lanes/:id/preflight?action=reset|remove|purge — read the
* action-specific facts that a destructive request must echo as `expect`.
+213
View File
@@ -0,0 +1,213 @@
/**
* @file healthScore.ts
* @description The Dashboard System Health score: a list of checks, each read
* against two thresholds (`warnAt` → amber, `badAt` → red) and costing up to
* `max` points as it climbs between them; the score is 100 minus every
* deduction (never below 0). Most of the weight is on what makes the machine
* or the dashboard feel slow — the event loop's p99 delay
* (`/api/settings/info` `server.event_loop`) and Linux pressure stall
* information for CPU, memory and disk I/O (`/api/host` `pressure`; the CPU
* falls back to load per core without PSI) — then RAM, the data disk, the V8
* heap, hooks going silent while agents work, and agent errors in the last
* 24 h. A check whose input is missing is `na` and costs nothing. Pure — the
* card in `Dashboard.tsx` only renders the result.
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
*/
import type { HostStats } from "./api";
export type CheckLevel = "ok" | "warn" | "bad" | "na";
export type CheckKey =
"loop" | "cpu" | "memory" | "io" | "ram" | "disk" | "heap" | "hooks" | "agents";
export interface HealthCheck {
key: CheckKey;
/** "lag" checks are what a user feels as a slow machine or dashboard. */
group: "lag" | "system";
/** The measured value in `unit`; null when not measured. */
value: number | null;
unit: "ms" | "%" | "min";
level: CheckLevel;
warnAt: number;
badAt: number;
/** Points this check took off the score. */
penalty: number;
/** The most it can take. */
max: number;
/** Extra facts for the row (e.g. which mount, how many agents). */
detail?: Record<string, string | number>;
}
export interface HealthInput {
/** `server.event_loop` (ms); absent on an older server. */
eventLoop?: { p50: number; p99: number; max: number };
heap: { heapUsed: number; heapTotal: number; heapLimit?: number };
hooksInstalled: boolean;
activity?: {
last_event_at: string | null;
working_agents: number;
agents_24h: { completed: number; error: number };
};
/** The database file, to find the disk it lives on; null for a developer. */
dbPath: string | null;
host: HostStats | null;
now?: number;
}
/** Fewest finished agents before an error rate means anything. */
const MIN_AGENTS = 5;
function check(
key: CheckKey,
group: HealthCheck["group"],
value: number | null,
unit: HealthCheck["unit"],
warnAt: number,
badAt: number,
max: number,
detail?: HealthCheck["detail"]
): HealthCheck {
if (value == null || !Number.isFinite(value)) {
return { key, group, value: null, unit, level: "na", warnAt, badAt, penalty: 0, max, detail };
}
const level: CheckLevel = value >= badAt ? "bad" : value >= warnAt ? "warn" : "ok";
const t = Math.max(0, Math.min(1, (value - warnAt) / (badAt - warnAt)));
return {
key,
group,
value,
unit,
level,
warnAt,
badAt,
penalty: Math.round(max * t * 10) / 10,
max,
detail,
};
}
/** The mount holding `path`: the longest mount point that prefixes it, else `/`. */
function diskOf(host: HostStats, path: string | null) {
const disks = host.disks ?? [];
if (path) {
const hit = disks
.filter((d) => path === d.mount || path.startsWith(d.mount === "/" ? "/" : `${d.mount}/`))
.sort((a, b) => b.mount.length - a.mount.length)[0];
if (hit) return hit;
}
return disks.find((d) => d.mount === "/") ?? disks[0] ?? null;
}
export function healthChecks(input: HealthInput): HealthCheck[] {
const { host } = input;
const now = input.now ?? Date.now();
const psi = host?.pressure ?? null;
const out: HealthCheck[] = [];
// ── What lag is made of ──
out.push(check("loop", "lag", input.eventLoop?.p99 ?? null, "ms", 100, 1000, 25));
if (psi?.cpu) {
out.push(check("cpu", "lag", psi.cpu.some10, "%", 20, 60, 20, { source: "psi" }));
} else {
const load =
host && host.cpu.cores > 0 ? ((host.cpu.load[0] ?? 0) / host.cpu.cores) * 100 : null;
out.push(check("cpu", "lag", load, "%", 100, 200, 20, { source: "load" }));
}
out.push(check("memory", "lag", psi?.memory?.some10 ?? null, "%", 5, 25, 25));
out.push(check("io", "lag", psi?.io?.some10 ?? null, "%", 15, 50, 20));
// ── Capacity and the dashboard's own signals ──
const mem = host?.memory;
out.push(
check(
"ram",
"system",
mem && mem.total > 0 ? (mem.used / mem.total) * 100 : null,
"%",
90,
98,
10
)
);
const disk = host ? diskOf(host, input.dbPath) : null;
out.push(
check(
"disk",
"system",
disk && disk.total > 0 ? (disk.used / disk.total) * 100 : null,
"%",
90,
98,
15,
disk ? { mount: disk.mount } : undefined
)
);
const heapMax = input.heap.heapLimit || input.heap.heapTotal;
out.push(
check(
"heap",
"system",
heapMax > 0 ? (input.heap.heapUsed / heapMax) * 100 : null,
"%",
70,
90,
15
)
);
// Hooks: not installed is always red; installed, they matter only while an
// agent works — then minutes of silence mean the dashboard went blind.
const a = input.activity;
if (!input.hooksInstalled) {
out.push({
key: "hooks",
group: "system",
value: null,
unit: "min",
level: "bad",
warnAt: 10,
badAt: 30,
penalty: 30,
max: 30,
detail: { state: "missing" },
});
} else if (!a || a.working_agents === 0 || !a.last_event_at) {
out.push({
...check("hooks", "system", null, "min", 10, 30, 30, { state: "idle" }),
level: "ok",
});
} else {
const age = Math.max(0, (now - Date.parse(a.last_event_at)) / 60000);
out.push(
check("hooks", "system", age, "min", 10, 30, 30, {
state: "working",
working: a.working_agents,
})
);
}
const done = a ? a.agents_24h.completed + a.agents_24h.error : 0;
out.push(
check(
"agents",
"system",
a && done >= MIN_AGENTS ? (a.agents_24h.error / done) * 100 : null,
"%",
10,
30,
10,
a ? { error: a.agents_24h.error, total: done } : undefined
)
);
return out;
}
export function healthScore(checks: HealthCheck[]): number {
const lost = checks.reduce((s, c) => s + c.penalty, 0);
return Math.max(0, Math.min(100, Math.round((100 - lost) * 10) / 10));
}
/** Green from 90, amber from 70, red below. */
export function scoreLevel(score: number): Exclude<CheckLevel, "na"> {
return score >= 90 ? "ok" : score >= 70 ? "warn" : "bad";
}
+9
View File
@@ -746,6 +746,15 @@ export interface Session {
/** ISO time the user pinned the session to the top of its workspace's
* sidebar list; null when not pinned. Maps to `sessions.pinned_at`. */
pinned_at?: string | null;
/** Server mode, `GET /api/sessions` only: the account owning the session's
* workspace (null when no workspace owns it). Absent in local mode. */
owner_id?: number | null;
/** That owner's username (`users.username`). */
owner_name?: string | null;
/** That owner's display name (`users.display_name`); null when none is set. */
owner_display_name?: string | null;
/** When that owner's avatar was last set (`users.avatar_at`); null = none. */
owner_avatar_at?: string | null;
}
/**