feat(health): System Health in three groups, a score that sees lag; admin user detail

Dashboard → System Health is grouped into AgentHub, Machine and Agent
activity. The machine (GET /api/host, every signed-in user) is drawn by
components/health/HostHealth.tsx: host card beside the CPU/memory history,
then CPU (threads as compact cells + temperatures/GPUs), disks & network,
busiest processes, and per-user tiles. Concurrency bars each show their own
agent type on hover.

The Health Score (client/src/lib/healthScore.ts) is now 100 minus what each
check costs, between an amber and a red threshold: event-loop p99
(server/lib/loop-delay.js, shared with /api/metrics), Linux PSI pressure for
CPU, memory and disk I/O (/api/host `pressure`; load per core without PSI),
RAM, the database's disk, V8 heap, hooks silent while agents work, and agent
errors in the last 24 h. All-time success rate, API errors and the transcript
cache hit rate no longer count. /api/settings/info adds server.event_loop and
activity {last_event_at, working_agents, agents_24h}.

Administration: a user row opens its detail in place (GET
/api/users/:id/details — spend, account pool share, live panes, workspaces,
newest sessions); the server panel is gone, member tmux folded into the table;
the Sessions list shows each row's owner in server mode.
This commit is contained in:
2026-10-27 16:39:38 +07:00
parent df15789982
commit 1c6dfea5b5
49 changed files with 5154 additions and 1971 deletions
+2 -2
View File
@@ -104,8 +104,8 @@ const STAFF_ONLY_PREFIXES = [
"/alerts/rules",
"/users",
"/accounts",
// The machine's hardware and every user's CPU/memory share.
"/host",
// `/host` (the machine and each user's CPU/memory share) is NOT here: the
// Dashboard's System Health tab shows it to every signed-in user.
];
// Staff-only paths with an id in the middle, matched on `apiPath()`'s
// lower-cased form: the host's own AI-tool config files (they may hold keys).
+383 -34
View File
@@ -1,7 +1,7 @@
/**
* @file host-stats.js
* @description The Administration page's Server tab (`GET /api/host`, staff
* only): the machine's hardware and load, and how much of its CPU and memory
* @description The Dashboard System Health tab's machine cards (`GET /api/host`,
* every signed-in user): the machine's hardware and load, and how much of its CPU and memory
* each AgentHub user is using right now. Read-only — nothing here signals,
* renices or kills a process.
*
@@ -17,9 +17,22 @@
* children carry it too); else `host` (the dashboard and everything else the
* host account runs) or `system` (root and every other account). Only the
* kernel's `comm` name is reported for a process — never its argv or env,
* which can hold a prompt or a key. Every read is async, and one sample is
* which can hold a prompt or a key. The same sample window also gives per-core
* CPU % (`/proc/stat` cpuN), network rx/tx per interface (`/proc/net/dev`, no
* `lo`), disk read/write per whole disk (`/proc/diskstats`), and the ten
* busiest processes machine-wide with their group; temperatures come from
* `/sys/class/hwmon`, physical cores from `/proc/cpuinfo`, and the dashboard's
* own pid/Node/uptime/RSS from `process`, and pressure stall information
* (`/proc/pressure/{cpu,memory,io}`, PSI — the share of time tasks waited on
* that resource, i.e. how laggy the machine feels; `null` on a kernel without
* it). Every read is async, and one sample is
* shared by concurrent callers and cached for `FRESH_MS`. Linux only: other
* platforms get the `os` facts with `usage: null`.
* platforms get the `os` facts with `usage: null` and empty lists.
*
* History (`GET /api/host/history`): `startHistory` (every server on Linux,
* server/index.js) keeps an in-memory ring of machine CPU/memory/swap/load1
* and per-user CPU/memory, one point every 30 s for 24 h, from the cached
* snapshot; it starts empty at every server start and is never written to disk.
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
*/
@@ -32,6 +45,10 @@ const { execFile } = require("node:child_process");
const SAMPLE_MS = 1000;
const FRESH_MS = 3000;
const TOP_PER_GROUP = 5;
const TOP_PROCESSES = 10;
const HISTORY_EVERY_MS = 30_000;
const HISTORY_KEEP_MS = 24 * 3600_000;
const HISTORY_MAX_POINTS = 360;
// File systems that hold data; pseudo and loop mounts are left out.
const REAL_FS = new Set([
"ext2",
@@ -71,13 +88,154 @@ async function memory() {
};
}
/** The machine's jiffies from `/proc/stat`'s `cpu` line: `{ total, idle }`. */
async function cpuTimes() {
const text = await readText("/proc/stat");
const line = text && text.split("\n").find((l) => l.startsWith("cpu "));
if (!line) return null;
const n = line.trim().split(/\s+/).slice(1).map(Number);
return { total: n.reduce((a, b) => a + b, 0), idle: n[3] + (n[4] || 0) };
/**
* `/proc/stat`'s jiffies: the machine's `cpu` line and each `cpuN` line, each
* `{ total, idle }` (idle = idle + iowait). `{ all: null, cores: [] }` when
* there is no `cpu` line.
*/
function parseCpuStat(text) {
const times = (line) => {
const n = line.trim().split(/\s+/).slice(1).map(Number);
return { total: n.reduce((a, b) => a + b, 0), idle: n[3] + (n[4] || 0) };
};
const lines = (text || "").split("\n");
const all = lines.find((l) => l.startsWith("cpu "));
return {
all: all ? times(all) : null,
cores: lines.filter((l) => /^cpu\d+ /.test(l)).map(times),
};
}
/**
* One `/proc/pressure/<resource>` file: `{some10, some60, full10}` — the % of
* the last 10 s / 60 s that at least one task (`some`) or every task (`full`)
* was stalled on it. `null` when the file is missing or unreadable.
*/
function parsePressure(text) {
if (!text) return null;
const line = (kind) => {
const m = new RegExp(`^${kind} avg10=([\\d.]+) avg60=([\\d.]+)`, "m").exec(text);
return m ? { avg10: Number(m[1]), avg60: Number(m[2]) } : null;
};
const some = line("some");
if (!some) return null;
return { some10: some.avg10, some60: some.avg60, full10: line("full")?.avg10 ?? 0 };
}
/** PSI for cpu, memory and io; `null` when the kernel has none of them. */
async function pressure() {
const [cpu, memory, io] = await Promise.all(
["cpu", "memory", "io"].map((r) => readText(`/proc/pressure/${r}`).then(parsePressure))
);
return cpu || memory || io ? { cpu, memory, io } : null;
}
/** Busy % between two `{ total, idle }` readings. */
const busyPct = (a, b) => {
const total = b.total - a.total;
return total > 0 ? round(((total - (b.idle - a.idle)) / total) * 100) : 0;
};
/** `/proc/net/dev`: `Map<iface, { rx, tx }>` in bytes, `lo` left out. */
function parseNetDev(text) {
const out = new Map();
for (const line of (text || "").split("\n").slice(2)) {
const i = line.indexOf(":");
if (i < 0) continue;
const name = line.slice(0, i).trim();
if (!name || name === "lo") continue;
const f = line
.slice(i + 1)
.trim()
.split(/\s+/)
.map(Number);
out.set(name, { rx: f[0] || 0, tx: f[8] || 0 });
}
return out;
}
/**
* `/proc/diskstats`: `Map<device, { read, write }>` in bytes (sectors × 512)
* for whole disks only (`isDisk(name)`), never loop/ram/zram/optical.
*/
function parseDiskstats(text, isDisk = () => true) {
const out = new Map();
for (const line of (text || "").split("\n")) {
const f = line.trim().split(/\s+/);
if (f.length < 10) continue;
const name = f[2];
if (/^(loop|ram|zram|sr|fd)\d*/.test(name) || !isDisk(name)) continue;
out.set(name, { read: Number(f[5]) * 512, write: Number(f[9]) * 512 });
}
return out;
}
/** Whole disks: the names under `/sys/block` (partitions are not there). */
function wholeDisks() {
try {
return new Set(fs.readdirSync("/sys/block"));
} catch {
return null;
}
}
/** Per-key `{a, b}` byte counters → per-second rates over `ms`. */
function rates(m0, m1, ms, keys) {
const sec = Math.max(0.001, ms / 1000);
const out = [];
for (const [name, v1] of m1) {
const v0 = m0.get(name);
if (!v0) continue;
const row = { name };
for (const k of keys) row[k] = Math.max(0, Math.round((v1[k] - v0[k]) / sec));
out.push(row);
}
return out;
}
/**
* Temperatures from `/sys/class/hwmon/*`: `[{ chip, label, celsius }]`, one
* per `tempN_input` (label from `tempN_label`, else `tempN`). [] when none.
*/
async function temperatures(root = "/sys/class/hwmon") {
let chips;
try {
chips = await fsp.readdir(root);
} catch {
return [];
}
const out = [];
for (const c of chips.sort()) {
const dir = path.join(root, c);
const chip = ((await readText(path.join(dir, "name"))) || c).trim();
let files;
try {
files = await fsp.readdir(dir);
} catch {
continue;
}
for (const f of files.filter((x) => /^temp\d+_input$/.test(x)).sort()) {
const raw = Number(((await readText(path.join(dir, f))) || "").trim());
if (!Number.isFinite(raw) || raw === 0) continue;
const id = f.replace("_input", "");
const label = ((await readText(path.join(dir, `${id}_label`))) || id).trim();
out.push({ chip, label, celsius: round(raw / 1000) });
}
}
return out;
}
/** Physical cores from `/proc/cpuinfo` (distinct physical id + core id), or null. */
function physicalCores(text) {
const seen = new Set();
let phys = "0";
for (const line of (text || "").split("\n")) {
const m = /^(physical id|core id)\s*:\s*(\d+)/.exec(line);
if (!m) continue;
if (m[1] === "physical id") phys = m[2];
else seen.add(`${phys}:${m[2]}`);
}
return seen.size || null;
}
/** `HDD`/`SSD` for a `/dev/...` device, or null when /sys cannot tell. */
@@ -269,14 +427,47 @@ let deps = {
wait: (ms) => new Promise((r) => setTimeout(r, ms)),
};
/** Per-user CPU and memory from two samples. */
async function usage() {
const t0 = await cpuTimes();
const p0 = await processes();
/** The counters one sample reads: CPU jiffies, network, disk I/O, processes. */
async function counters(isDisk) {
const [stat, net, io] = await Promise.all([
readText("/proc/stat"),
readText("/proc/net/dev"),
readText("/proc/diskstats"),
]);
return {
at: Date.now(),
cpu: parseCpuStat(stat),
net: parseNetDev(net),
io: parseDiskstats(io, isDisk),
procs: await processes(),
};
}
/**
* Two samples `SAMPLE_MS` apart: per-user CPU and memory (`usage`), and from
* the same window per-core CPU, network and disk I/O rates and the busiest
* processes machine-wide.
*/
async function sampleWindow(mem) {
const disksSet = wholeDisks();
const isDisk = (n) => !disksSet || disksSet.has(n);
const s0 = await counters(isDisk);
await deps.wait(SAMPLE_MS);
const t1 = await cpuTimes();
const p1 = await processes();
const mem = await memory();
const s1 = await counters(isDisk);
const ms = s1.at - s0.at;
const cores = s1.cpu.cores.map((c, i) => (s0.cpu.cores[i] ? busyPct(s0.cpu.cores[i], c) : 0));
const use = await usage(s0.cpu.all, s1.cpu.all, s0.procs, s1.procs, mem);
return {
usage: use ? use.usage : null,
topProcesses: use ? use.top : [],
cores,
network: rates(s0.net, s1.net, ms, ["rx", "tx"]),
diskIo: rates(s0.io, s1.io, ms, ["read", "write"]),
};
}
/** Per-user CPU and memory from two samples (`usage`) and the busiest processes (`top`). */
async function usage(t0, t1, p0, p1, mem) {
if (!t0 || !t1) return null;
const totalDelta = Math.max(1, t1.total - t0.total);
const hostUid = typeof process.getuid === "function" ? process.getuid() : -1;
@@ -286,6 +477,7 @@ async function usage() {
const byId = new Map(users.map((u) => [u.id, u]));
const ownerCache = new Map();
const groups = new Map();
const all = [];
const group = (key, extra) => {
if (!groups.has(key))
groups.set(key, { key, cpu: 0, memory: 0, processes: 0, top: [], ...extra });
@@ -301,7 +493,7 @@ async function usage() {
const member = deps.memberNameForUid(p.uid);
if (member && byUsername.has(member)) {
const u = byUsername.get(member);
g = group(`user:${u.id}`, { kind: "user", userId: u.id, username: u.username });
g = group(`user:${u.id}`, userExtra(u));
} else if (p.uid === hostUid) {
const laneId = await laneIdOf(pid);
let ownerId = null;
@@ -318,9 +510,7 @@ async function usage() {
ownerId = ownerCache.get(laneId);
}
const u = ownerId != null ? byId.get(ownerId) : null;
g = u
? group(`user:${u.id}`, { kind: "user", userId: u.id, username: u.username })
: group("host", { kind: "host" });
g = u ? group(`user:${u.id}`, userExtra(u)) : group("host", { kind: "host" });
} else {
g = group("system", { kind: "system" });
}
@@ -329,6 +519,7 @@ async function usage() {
g.memory += p.rss;
g.processes += 1;
g.top.push({ pid, name: p.comm, cpu, memory: p.rss });
all.push({ pid, name: p.comm, cpu, memory: p.rss, group: g.key });
})
);
@@ -342,9 +533,7 @@ async function usage() {
if (!groups.has(`user:${u.id}`))
list.push({
key: `user:${u.id}`,
kind: "user",
userId: u.id,
username: u.username,
...userExtra(u),
cpu: 0,
memory: 0,
memoryPercent: 0,
@@ -353,24 +542,40 @@ async function usage() {
});
}
list.sort((a, b) => b.cpu - a.cpu || b.memory - a.memory);
all.sort((a, b) => b.cpu - a.cpu || b.memory - a.memory);
return {
cpu: round(((totalDelta - (t1.idle - t0.idle)) / totalDelta) * 100),
sampleMs: SAMPLE_MS,
groups: list,
usage: {
cpu: round(((totalDelta - (t1.idle - t0.idle)) / totalDelta) * 100),
sampleMs: SAMPLE_MS,
groups: list,
},
top: all.slice(0, TOP_PROCESSES).map((t) => ({ ...t, cpu: round(t.cpu) })),
};
}
const round = (n) => Math.round(n * 10) / 10;
/** A user's group fields: id, username and display name (every viewer reads them). */
const userExtra = (u) => ({
kind: "user",
userId: u.id,
username: u.username,
displayName: u.displayName ?? null,
});
async function collect() {
const linux = process.platform === "linux";
const cpus = os.cpus();
const [mem, diskList, gpuList, use] = await Promise.all([
memory(),
const mem = await memory();
const [diskList, gpuList, win, temps, cpuinfo, psi] = await Promise.all([
linux ? disks() : [],
linux ? gpus() : [],
linux ? usage() : null,
linux ? sampleWindow(mem) : null,
linux ? temperatures() : [],
linux ? readText("/proc/cpuinfo") : null,
linux ? pressure() : null,
]);
const pmem = process.memoryUsage();
return {
at: new Date().toISOString(),
hostname: os.hostname(),
@@ -381,12 +586,31 @@ async function collect() {
arch: os.arch(),
},
uptime: os.uptime(),
cpu: { model: cpus[0]?.model?.trim() || null, cores: cpus.length, load: os.loadavg() },
bootTime: new Date(Date.now() - os.uptime() * 1000).toISOString(),
cpu: {
model: cpus[0]?.model?.trim() || null,
cores: cpus.length,
load: os.loadavg(),
physicalCores: physicalCores(cpuinfo),
perCore: win ? win.cores : [],
},
memory: mem,
disks: diskList,
homesMount: mountOf(diskList, deps.homesRoot()),
gpus: gpuList,
usage: use,
usage: win ? win.usage : null,
network: win ? win.network : [],
diskIo: win ? win.diskIo : [],
temperatures: temps,
pressure: psi,
topProcesses: win ? win.topProcesses : [],
process: {
pid: process.pid,
node: process.version,
uptime: Math.round(process.uptime()),
rss: pmem.rss,
heapUsed: pmem.heapUsed,
},
};
}
@@ -409,9 +633,134 @@ async function hostStats() {
return inflight;
}
// ── History ────────────────────────────────────────────────────────────────
// An in-memory ring of machine-level samples, one every HISTORY_EVERY_MS for
// HISTORY_KEEP_MS, started by every server on Linux (server/index.js). It
// starts empty at every server start: nothing is written to disk.
let history = [];
let historyTimer = null;
/** One history point from a snapshot. */
function historyPoint(s) {
const users = {};
for (const g of s.usage?.groups || []) {
if (g.kind === "user" && (g.cpu || g.memory))
users[g.userId] = { cpu: g.cpu, memory: g.memory };
}
return {
at: s.at,
cpu: s.usage ? s.usage.cpu : null,
memory: round((s.memory.used / Math.max(1, s.memory.total)) * 100),
memoryUsed: s.memory.used,
swapUsed: s.memory.swapUsed,
load1: round(s.cpu.load[0] || 0),
users,
};
}
/** Take one sample now (reusing a fresh snapshot) and drop what is too old. */
async function sampleHistory() {
try {
const point = historyPoint(await hostStats());
const cutoff = Date.now() - HISTORY_KEEP_MS;
history = [...history.filter((p) => Date.parse(p.at) >= cutoff), point];
} catch {
/* a failed sample is a gap, never a crash */
}
}
/** Start the sampler (Linux only; idempotent). The timer never holds the process open. */
function startHistory() {
if (historyTimer || process.platform !== "linux") return;
void sampleHistory();
historyTimer = setInterval(() => void sampleHistory(), HISTORY_EVERY_MS);
historyTimer.unref?.();
}
function stopHistory() {
if (historyTimer) clearInterval(historyTimer);
historyTimer = null;
}
const RANGES = { "1h": 3600_000, "6h": 6 * 3600_000, "24h": 24 * 3600_000 };
/**
* The points of the last `range` (`1h`, `6h`, `24h`; else 1h), averaged down
* to at most HISTORY_MAX_POINTS. `startedAt` = the oldest point kept, so the
* UI can say the history begins there.
*/
function hostHistory(range) {
const span = RANGES[range] || RANGES["1h"];
const since = Date.now() - span;
const pts = history.filter((p) => Date.parse(p.at) >= since);
return {
range: RANGES[range] ? range : "1h",
intervalMs: HISTORY_EVERY_MS,
running: historyTimer != null,
startedAt: history[0]?.at ?? null,
points: downsample(pts, HISTORY_MAX_POINTS),
};
}
/** Average consecutive points into at most `max` buckets (numbers only; `at` = the bucket's last). */
function downsample(pts, max) {
if (pts.length <= max) return pts;
const per = Math.ceil(pts.length / max);
const out = [];
for (let i = 0; i < pts.length; i += per) {
const b = pts.slice(i, i + per);
const avg = (k) => {
const v = b.map((p) => p[k]).filter((x) => x != null);
return v.length ? round(v.reduce((a, x) => a + x, 0) / v.length) : null;
};
const users = {};
for (const p of b)
for (const [id, u] of Object.entries(p.users)) {
const t = (users[id] ||= { cpu: 0, memory: 0 });
t.cpu += u.cpu / b.length;
t.memory += u.memory / b.length;
}
for (const u of Object.values(users)) {
u.cpu = round(u.cpu);
u.memory = Math.round(u.memory);
}
out.push({
at: b[b.length - 1].at,
cpu: avg("cpu"),
memory: avg("memory"),
memoryUsed: Math.round(avg("memoryUsed") ?? 0),
swapUsed: Math.round(avg("swapUsed") ?? 0),
load1: avg("load1"),
users,
});
}
return out;
}
function __setDeps(over) {
deps = { ...deps, ...over };
cache = null;
}
module.exports = { hostStats, mountOf, __setDeps };
/** Tests: replace the history ring. */
function __setHistory(points) {
history = points;
}
module.exports = {
hostStats,
hostHistory,
startHistory,
stopHistory,
mountOf,
parseCpuStat,
parsePressure,
parseNetDev,
parseDiskstats,
physicalCores,
temperatures,
downsample,
__setDeps,
__setHistory,
};
+36
View File
@@ -0,0 +1,36 @@
/**
* @file loop-delay.js
* @description How long this server's event loop was held up — what an
* operator sees as a laggy dashboard or terminal: every request, hook and
* keystroke echo waits for it. One `monitorEventLoopDelay` histogram for the
* whole process, sampled every 20 ms and reset each minute, so `loopDelay()`
* describes the last full minute (the running one until the first minute
* ends). Read by `GET /api/metrics` (Prometheus gauges) and
* `GET /api/settings/info` (the System Health score).
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
*/
const { monitorEventLoopDelay } = require("perf_hooks");
const WINDOW_MS = 60_000;
const histogram = monitorEventLoopDelay({ resolution: 20 });
histogram.enable();
const read = () => ({
p50: histogram.percentile(50),
p99: histogram.percentile(99),
max: histogram.max,
});
let lastWindow = null; // the last full minute, reported until the next one ends
setInterval(() => {
lastWindow = read();
histogram.reset();
}, WINDOW_MS).unref();
/** `{p50, p99, max}` in nanoseconds over the last minute. */
function loopDelay() {
return lastWindow || read();
}
module.exports = { loopDelay, WINDOW_MS };
+16 -7
View File
@@ -8,7 +8,9 @@
* A Claude session counts only for the hours one of THIS account's runs held
* it (start hour to end hour), so a session resumed on another account is
* not booked twice; Codex has no hourly rows, so its session counts whole
* once a run of the account started in the window, as before.
* once a run of the account started in the window, as before. With `ownerId`
* only the runs of that user's workspaces count — one user's share of the
* account, for the Administration page's user detail.
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
*/
@@ -19,21 +21,28 @@ const tokensOf = (r) =>
/**
* `{spend5h, spend7d, tokens5h, tokens7d}` of `account` on `tool`. With
* `wholeSession`, a session's every row counts in a window its run started in.
* `wholeSession`, a session's every row counts in a window its run started in;
* with `ownerId`, only runs on that user's workspaces.
*/
function accountUsage(tool, account, { wholeSession = false, now = Date.now() } = {}) {
function accountUsage(
tool,
account,
{ wholeSession = false, now = Date.now(), ownerId = null } = {}
) {
const { db, stmts } = require("../db");
const { calculateCost, hourlyTokenRows } = require("../routes/pricing");
const since7d = new Date(now - WINDOWS["7d"] * 3600e3).toISOString();
const owned = ownerId == null ? "" : "AND lane_id IN (SELECT id FROM lanes WHERE owner_id = ?)";
const ownedArgs = ownerId == null ? [] : [ownerId];
const runs = db
.prepare(
`SELECT session_id, substr(started_at, 1, 13) AS start_hour, started_at,
substr(COALESCE(ended_at, '9999'), 1, 13) AS end_hour
FROM dashboard_runs
WHERE tool = ? AND account = ? AND session_id IS NOT NULL
AND COALESCE(ended_at, '9999') >= ?`
AND COALESCE(ended_at, '9999') >= ? ${owned}`
)
.all(tool, account, since7d);
.all(tool, account, since7d, ...ownedArgs);
const out = { spend5h: 0, spend7d: 0, tokens5h: 0, tokens7d: 0 };
if (runs.length === 0) return out;
const runsOf = new Map();
@@ -41,8 +50,8 @@ function accountUsage(tool, account, { wholeSession = false, now = Date.now() }
const rows = hourlyTokenRows(
`WHERE s.id IN (SELECT session_id FROM dashboard_runs
WHERE tool = ? AND account = ? AND session_id IS NOT NULL
AND COALESCE(ended_at, '9999') >= ?)`,
[tool, account, since7d]
AND COALESCE(ended_at, '9999') >= ? ${owned})`,
[tool, account, since7d, ...ownedArgs]
);
const rules = stmts.listPricing.all();
for (const [label, hours] of Object.entries(WINDOWS)) {