1c6dfea5b5
Dashboard → System Health is grouped into AgentHub, Machine and Agent
activity. The machine (GET /api/host, every signed-in user) is drawn by
components/health/HostHealth.tsx: host card beside the CPU/memory history,
then CPU (threads as compact cells + temperatures/GPUs), disks & network,
busiest processes, and per-user tiles. Concurrency bars each show their own
agent type on hover.
The Health Score (client/src/lib/healthScore.ts) is now 100 minus what each
check costs, between an amber and a red threshold: event-loop p99
(server/lib/loop-delay.js, shared with /api/metrics), Linux PSI pressure for
CPU, memory and disk I/O (/api/host `pressure`; load per core without PSI),
RAM, the database's disk, V8 heap, hooks silent while agents work, and agent
errors in the last 24 h. All-time success rate, API errors and the transcript
cache hit rate no longer count. /api/settings/info adds server.event_loop and
activity {last_event_at, working_agents, agents_24h}.
Administration: a user row opens its detail in place (GET
/api/users/:id/details — spend, account pool share, live panes, workspaces,
newest sessions); the server panel is gone, member tmux folded into the table;
the Sessions list shows each row's owner in server mode.
214 lines
6.4 KiB
TypeScript
214 lines
6.4 KiB
TypeScript
/**
|
|
* @file healthScore.ts
|
|
* @description The Dashboard System Health score: a list of checks, each read
|
|
* against two thresholds (`warnAt` → amber, `badAt` → red) and costing up to
|
|
* `max` points as it climbs between them; the score is 100 minus every
|
|
* deduction (never below 0). Most of the weight is on what makes the machine
|
|
* or the dashboard feel slow — the event loop's p99 delay
|
|
* (`/api/settings/info` `server.event_loop`) and Linux pressure stall
|
|
* information for CPU, memory and disk I/O (`/api/host` `pressure`; the CPU
|
|
* falls back to load per core without PSI) — then RAM, the data disk, the V8
|
|
* heap, hooks going silent while agents work, and agent errors in the last
|
|
* 24 h. A check whose input is missing is `na` and costs nothing. Pure — the
|
|
* card in `Dashboard.tsx` only renders the result.
|
|
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
|
|
*/
|
|
|
|
import type { HostStats } from "./api";
|
|
|
|
export type CheckLevel = "ok" | "warn" | "bad" | "na";
|
|
export type CheckKey =
|
|
"loop" | "cpu" | "memory" | "io" | "ram" | "disk" | "heap" | "hooks" | "agents";
|
|
|
|
export interface HealthCheck {
|
|
key: CheckKey;
|
|
/** "lag" checks are what a user feels as a slow machine or dashboard. */
|
|
group: "lag" | "system";
|
|
/** The measured value in `unit`; null when not measured. */
|
|
value: number | null;
|
|
unit: "ms" | "%" | "min";
|
|
level: CheckLevel;
|
|
warnAt: number;
|
|
badAt: number;
|
|
/** Points this check took off the score. */
|
|
penalty: number;
|
|
/** The most it can take. */
|
|
max: number;
|
|
/** Extra facts for the row (e.g. which mount, how many agents). */
|
|
detail?: Record<string, string | number>;
|
|
}
|
|
|
|
export interface HealthInput {
|
|
/** `server.event_loop` (ms); absent on an older server. */
|
|
eventLoop?: { p50: number; p99: number; max: number };
|
|
heap: { heapUsed: number; heapTotal: number; heapLimit?: number };
|
|
hooksInstalled: boolean;
|
|
activity?: {
|
|
last_event_at: string | null;
|
|
working_agents: number;
|
|
agents_24h: { completed: number; error: number };
|
|
};
|
|
/** The database file, to find the disk it lives on; null for a developer. */
|
|
dbPath: string | null;
|
|
host: HostStats | null;
|
|
now?: number;
|
|
}
|
|
|
|
/** Fewest finished agents before an error rate means anything. */
|
|
const MIN_AGENTS = 5;
|
|
|
|
function check(
|
|
key: CheckKey,
|
|
group: HealthCheck["group"],
|
|
value: number | null,
|
|
unit: HealthCheck["unit"],
|
|
warnAt: number,
|
|
badAt: number,
|
|
max: number,
|
|
detail?: HealthCheck["detail"]
|
|
): HealthCheck {
|
|
if (value == null || !Number.isFinite(value)) {
|
|
return { key, group, value: null, unit, level: "na", warnAt, badAt, penalty: 0, max, detail };
|
|
}
|
|
const level: CheckLevel = value >= badAt ? "bad" : value >= warnAt ? "warn" : "ok";
|
|
const t = Math.max(0, Math.min(1, (value - warnAt) / (badAt - warnAt)));
|
|
return {
|
|
key,
|
|
group,
|
|
value,
|
|
unit,
|
|
level,
|
|
warnAt,
|
|
badAt,
|
|
penalty: Math.round(max * t * 10) / 10,
|
|
max,
|
|
detail,
|
|
};
|
|
}
|
|
|
|
/** The mount holding `path`: the longest mount point that prefixes it, else `/`. */
|
|
function diskOf(host: HostStats, path: string | null) {
|
|
const disks = host.disks ?? [];
|
|
if (path) {
|
|
const hit = disks
|
|
.filter((d) => path === d.mount || path.startsWith(d.mount === "/" ? "/" : `${d.mount}/`))
|
|
.sort((a, b) => b.mount.length - a.mount.length)[0];
|
|
if (hit) return hit;
|
|
}
|
|
return disks.find((d) => d.mount === "/") ?? disks[0] ?? null;
|
|
}
|
|
|
|
export function healthChecks(input: HealthInput): HealthCheck[] {
|
|
const { host } = input;
|
|
const now = input.now ?? Date.now();
|
|
const psi = host?.pressure ?? null;
|
|
const out: HealthCheck[] = [];
|
|
|
|
// ── What lag is made of ──
|
|
out.push(check("loop", "lag", input.eventLoop?.p99 ?? null, "ms", 100, 1000, 25));
|
|
if (psi?.cpu) {
|
|
out.push(check("cpu", "lag", psi.cpu.some10, "%", 20, 60, 20, { source: "psi" }));
|
|
} else {
|
|
const load =
|
|
host && host.cpu.cores > 0 ? ((host.cpu.load[0] ?? 0) / host.cpu.cores) * 100 : null;
|
|
out.push(check("cpu", "lag", load, "%", 100, 200, 20, { source: "load" }));
|
|
}
|
|
out.push(check("memory", "lag", psi?.memory?.some10 ?? null, "%", 5, 25, 25));
|
|
out.push(check("io", "lag", psi?.io?.some10 ?? null, "%", 15, 50, 20));
|
|
|
|
// ── Capacity and the dashboard's own signals ──
|
|
const mem = host?.memory;
|
|
out.push(
|
|
check(
|
|
"ram",
|
|
"system",
|
|
mem && mem.total > 0 ? (mem.used / mem.total) * 100 : null,
|
|
"%",
|
|
90,
|
|
98,
|
|
10
|
|
)
|
|
);
|
|
const disk = host ? diskOf(host, input.dbPath) : null;
|
|
out.push(
|
|
check(
|
|
"disk",
|
|
"system",
|
|
disk && disk.total > 0 ? (disk.used / disk.total) * 100 : null,
|
|
"%",
|
|
90,
|
|
98,
|
|
15,
|
|
disk ? { mount: disk.mount } : undefined
|
|
)
|
|
);
|
|
const heapMax = input.heap.heapLimit || input.heap.heapTotal;
|
|
out.push(
|
|
check(
|
|
"heap",
|
|
"system",
|
|
heapMax > 0 ? (input.heap.heapUsed / heapMax) * 100 : null,
|
|
"%",
|
|
70,
|
|
90,
|
|
15
|
|
)
|
|
);
|
|
|
|
// Hooks: not installed is always red; installed, they matter only while an
|
|
// agent works — then minutes of silence mean the dashboard went blind.
|
|
const a = input.activity;
|
|
if (!input.hooksInstalled) {
|
|
out.push({
|
|
key: "hooks",
|
|
group: "system",
|
|
value: null,
|
|
unit: "min",
|
|
level: "bad",
|
|
warnAt: 10,
|
|
badAt: 30,
|
|
penalty: 30,
|
|
max: 30,
|
|
detail: { state: "missing" },
|
|
});
|
|
} else if (!a || a.working_agents === 0 || !a.last_event_at) {
|
|
out.push({
|
|
...check("hooks", "system", null, "min", 10, 30, 30, { state: "idle" }),
|
|
level: "ok",
|
|
});
|
|
} else {
|
|
const age = Math.max(0, (now - Date.parse(a.last_event_at)) / 60000);
|
|
out.push(
|
|
check("hooks", "system", age, "min", 10, 30, 30, {
|
|
state: "working",
|
|
working: a.working_agents,
|
|
})
|
|
);
|
|
}
|
|
|
|
const done = a ? a.agents_24h.completed + a.agents_24h.error : 0;
|
|
out.push(
|
|
check(
|
|
"agents",
|
|
"system",
|
|
a && done >= MIN_AGENTS ? (a.agents_24h.error / done) * 100 : null,
|
|
"%",
|
|
10,
|
|
30,
|
|
10,
|
|
a ? { error: a.agents_24h.error, total: done } : undefined
|
|
)
|
|
);
|
|
return out;
|
|
}
|
|
|
|
export function healthScore(checks: HealthCheck[]): number {
|
|
const lost = checks.reduce((s, c) => s + c.penalty, 0);
|
|
return Math.max(0, Math.min(100, Math.round((100 - lost) * 10) / 10));
|
|
}
|
|
|
|
/** Green from 90, amber from 70, red below. */
|
|
export function scoreLevel(score: number): Exclude<CheckLevel, "na"> {
|
|
return score >= 90 ? "ok" : score >= 70 ? "warn" : "bad";
|
|
}
|