feat(health): System Health in three groups, a score that sees lag; admin user detail
Dashboard → System Health is grouped into AgentHub, Machine and Agent
activity. The machine (GET /api/host, every signed-in user) is drawn by
components/health/HostHealth.tsx: host card beside the CPU/memory history,
then CPU (threads as compact cells + temperatures/GPUs), disks & network,
busiest processes, and per-user tiles. Concurrency bars each show their own
agent type on hover.
The Health Score (client/src/lib/healthScore.ts) is now 100 minus what each
check costs, between an amber and a red threshold: event-loop p99
(server/lib/loop-delay.js, shared with /api/metrics), Linux PSI pressure for
CPU, memory and disk I/O (/api/host `pressure`; load per core without PSI),
RAM, the database's disk, V8 heap, hooks silent while agents work, and agent
errors in the last 24 h. All-time success rate, API errors and the transcript
cache hit rate no longer count. /api/settings/info adds server.event_loop and
activity {last_event_at, working_agents, agents_24h}.
Administration: a user row opens its detail in place (GET
/api/users/:id/details — spend, account pool share, live panes, workspaces,
newest sessions); the server panel is gone, member tmux folded into the table;
the Sessions list shows each row's owner in server mode.
This commit is contained in:
+2
-2
@@ -104,8 +104,8 @@ const STAFF_ONLY_PREFIXES = [
|
||||
"/alerts/rules",
|
||||
"/users",
|
||||
"/accounts",
|
||||
// The machine's hardware and every user's CPU/memory share.
|
||||
"/host",
|
||||
// `/host` (the machine and each user's CPU/memory share) is NOT here: the
|
||||
// Dashboard's System Health tab shows it to every signed-in user.
|
||||
];
|
||||
// Staff-only paths with an id in the middle, matched on `apiPath()`'s
|
||||
// lower-cased form: the host's own AI-tool config files (they may hold keys).
|
||||
|
||||
+383
-34
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* @file host-stats.js
|
||||
* @description The Administration page's Server tab (`GET /api/host`, staff
|
||||
* only): the machine's hardware and load, and how much of its CPU and memory
|
||||
* @description The Dashboard System Health tab's machine cards (`GET /api/host`,
|
||||
* every signed-in user): the machine's hardware and load, and how much of its CPU and memory
|
||||
* each AgentHub user is using right now. Read-only — nothing here signals,
|
||||
* renices or kills a process.
|
||||
*
|
||||
@@ -17,9 +17,22 @@
|
||||
* children carry it too); else `host` (the dashboard and everything else the
|
||||
* host account runs) or `system` (root and every other account). Only the
|
||||
* kernel's `comm` name is reported for a process — never its argv or env,
|
||||
* which can hold a prompt or a key. Every read is async, and one sample is
|
||||
* which can hold a prompt or a key. The same sample window also gives per-core
|
||||
* CPU % (`/proc/stat` cpuN), network rx/tx per interface (`/proc/net/dev`, no
|
||||
* `lo`), disk read/write per whole disk (`/proc/diskstats`), and the ten
|
||||
* busiest processes machine-wide with their group; temperatures come from
|
||||
* `/sys/class/hwmon`, physical cores from `/proc/cpuinfo`, and the dashboard's
|
||||
* own pid/Node/uptime/RSS from `process`, and pressure stall information
|
||||
* (`/proc/pressure/{cpu,memory,io}`, PSI — the share of time tasks waited on
|
||||
* that resource, i.e. how laggy the machine feels; `null` on a kernel without
|
||||
* it). Every read is async, and one sample is
|
||||
* shared by concurrent callers and cached for `FRESH_MS`. Linux only: other
|
||||
* platforms get the `os` facts with `usage: null`.
|
||||
* platforms get the `os` facts with `usage: null` and empty lists.
|
||||
*
|
||||
* History (`GET /api/host/history`): `startHistory` (every server on Linux,
|
||||
* server/index.js) keeps an in-memory ring of machine CPU/memory/swap/load1
|
||||
* and per-user CPU/memory, one point every 30 s for 24 h, from the cached
|
||||
* snapshot; it starts empty at every server start and is never written to disk.
|
||||
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
|
||||
*/
|
||||
|
||||
@@ -32,6 +45,10 @@ const { execFile } = require("node:child_process");
|
||||
const SAMPLE_MS = 1000;
|
||||
const FRESH_MS = 3000;
|
||||
const TOP_PER_GROUP = 5;
|
||||
const TOP_PROCESSES = 10;
|
||||
const HISTORY_EVERY_MS = 30_000;
|
||||
const HISTORY_KEEP_MS = 24 * 3600_000;
|
||||
const HISTORY_MAX_POINTS = 360;
|
||||
// File systems that hold data; pseudo and loop mounts are left out.
|
||||
const REAL_FS = new Set([
|
||||
"ext2",
|
||||
@@ -71,13 +88,154 @@ async function memory() {
|
||||
};
|
||||
}
|
||||
|
||||
/** The machine's jiffies from `/proc/stat`'s `cpu` line: `{ total, idle }`. */
|
||||
async function cpuTimes() {
|
||||
const text = await readText("/proc/stat");
|
||||
const line = text && text.split("\n").find((l) => l.startsWith("cpu "));
|
||||
if (!line) return null;
|
||||
const n = line.trim().split(/\s+/).slice(1).map(Number);
|
||||
return { total: n.reduce((a, b) => a + b, 0), idle: n[3] + (n[4] || 0) };
|
||||
/**
|
||||
* `/proc/stat`'s jiffies: the machine's `cpu` line and each `cpuN` line, each
|
||||
* `{ total, idle }` (idle = idle + iowait). `{ all: null, cores: [] }` when
|
||||
* there is no `cpu` line.
|
||||
*/
|
||||
function parseCpuStat(text) {
|
||||
const times = (line) => {
|
||||
const n = line.trim().split(/\s+/).slice(1).map(Number);
|
||||
return { total: n.reduce((a, b) => a + b, 0), idle: n[3] + (n[4] || 0) };
|
||||
};
|
||||
const lines = (text || "").split("\n");
|
||||
const all = lines.find((l) => l.startsWith("cpu "));
|
||||
return {
|
||||
all: all ? times(all) : null,
|
||||
cores: lines.filter((l) => /^cpu\d+ /.test(l)).map(times),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* One `/proc/pressure/<resource>` file: `{some10, some60, full10}` — the % of
|
||||
* the last 10 s / 60 s that at least one task (`some`) or every task (`full`)
|
||||
* was stalled on it. `null` when the file is missing or unreadable.
|
||||
*/
|
||||
function parsePressure(text) {
|
||||
if (!text) return null;
|
||||
const line = (kind) => {
|
||||
const m = new RegExp(`^${kind} avg10=([\\d.]+) avg60=([\\d.]+)`, "m").exec(text);
|
||||
return m ? { avg10: Number(m[1]), avg60: Number(m[2]) } : null;
|
||||
};
|
||||
const some = line("some");
|
||||
if (!some) return null;
|
||||
return { some10: some.avg10, some60: some.avg60, full10: line("full")?.avg10 ?? 0 };
|
||||
}
|
||||
|
||||
/** PSI for cpu, memory and io; `null` when the kernel has none of them. */
|
||||
async function pressure() {
|
||||
const [cpu, memory, io] = await Promise.all(
|
||||
["cpu", "memory", "io"].map((r) => readText(`/proc/pressure/${r}`).then(parsePressure))
|
||||
);
|
||||
return cpu || memory || io ? { cpu, memory, io } : null;
|
||||
}
|
||||
|
||||
/** Busy % between two `{ total, idle }` readings. */
|
||||
const busyPct = (a, b) => {
|
||||
const total = b.total - a.total;
|
||||
return total > 0 ? round(((total - (b.idle - a.idle)) / total) * 100) : 0;
|
||||
};
|
||||
|
||||
/** `/proc/net/dev`: `Map<iface, { rx, tx }>` in bytes, `lo` left out. */
|
||||
function parseNetDev(text) {
|
||||
const out = new Map();
|
||||
for (const line of (text || "").split("\n").slice(2)) {
|
||||
const i = line.indexOf(":");
|
||||
if (i < 0) continue;
|
||||
const name = line.slice(0, i).trim();
|
||||
if (!name || name === "lo") continue;
|
||||
const f = line
|
||||
.slice(i + 1)
|
||||
.trim()
|
||||
.split(/\s+/)
|
||||
.map(Number);
|
||||
out.set(name, { rx: f[0] || 0, tx: f[8] || 0 });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* `/proc/diskstats`: `Map<device, { read, write }>` in bytes (sectors × 512)
|
||||
* for whole disks only (`isDisk(name)`), never loop/ram/zram/optical.
|
||||
*/
|
||||
function parseDiskstats(text, isDisk = () => true) {
|
||||
const out = new Map();
|
||||
for (const line of (text || "").split("\n")) {
|
||||
const f = line.trim().split(/\s+/);
|
||||
if (f.length < 10) continue;
|
||||
const name = f[2];
|
||||
if (/^(loop|ram|zram|sr|fd)\d*/.test(name) || !isDisk(name)) continue;
|
||||
out.set(name, { read: Number(f[5]) * 512, write: Number(f[9]) * 512 });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Whole disks: the names under `/sys/block` (partitions are not there). */
|
||||
function wholeDisks() {
|
||||
try {
|
||||
return new Set(fs.readdirSync("/sys/block"));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Per-key `{a, b}` byte counters → per-second rates over `ms`. */
|
||||
function rates(m0, m1, ms, keys) {
|
||||
const sec = Math.max(0.001, ms / 1000);
|
||||
const out = [];
|
||||
for (const [name, v1] of m1) {
|
||||
const v0 = m0.get(name);
|
||||
if (!v0) continue;
|
||||
const row = { name };
|
||||
for (const k of keys) row[k] = Math.max(0, Math.round((v1[k] - v0[k]) / sec));
|
||||
out.push(row);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Temperatures from `/sys/class/hwmon/*`: `[{ chip, label, celsius }]`, one
|
||||
* per `tempN_input` (label from `tempN_label`, else `tempN`). [] when none.
|
||||
*/
|
||||
async function temperatures(root = "/sys/class/hwmon") {
|
||||
let chips;
|
||||
try {
|
||||
chips = await fsp.readdir(root);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
const out = [];
|
||||
for (const c of chips.sort()) {
|
||||
const dir = path.join(root, c);
|
||||
const chip = ((await readText(path.join(dir, "name"))) || c).trim();
|
||||
let files;
|
||||
try {
|
||||
files = await fsp.readdir(dir);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const f of files.filter((x) => /^temp\d+_input$/.test(x)).sort()) {
|
||||
const raw = Number(((await readText(path.join(dir, f))) || "").trim());
|
||||
if (!Number.isFinite(raw) || raw === 0) continue;
|
||||
const id = f.replace("_input", "");
|
||||
const label = ((await readText(path.join(dir, `${id}_label`))) || id).trim();
|
||||
out.push({ chip, label, celsius: round(raw / 1000) });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Physical cores from `/proc/cpuinfo` (distinct physical id + core id), or null. */
|
||||
function physicalCores(text) {
|
||||
const seen = new Set();
|
||||
let phys = "0";
|
||||
for (const line of (text || "").split("\n")) {
|
||||
const m = /^(physical id|core id)\s*:\s*(\d+)/.exec(line);
|
||||
if (!m) continue;
|
||||
if (m[1] === "physical id") phys = m[2];
|
||||
else seen.add(`${phys}:${m[2]}`);
|
||||
}
|
||||
return seen.size || null;
|
||||
}
|
||||
|
||||
/** `HDD`/`SSD` for a `/dev/...` device, or null when /sys cannot tell. */
|
||||
@@ -269,14 +427,47 @@ let deps = {
|
||||
wait: (ms) => new Promise((r) => setTimeout(r, ms)),
|
||||
};
|
||||
|
||||
/** Per-user CPU and memory from two samples. */
|
||||
async function usage() {
|
||||
const t0 = await cpuTimes();
|
||||
const p0 = await processes();
|
||||
/** The counters one sample reads: CPU jiffies, network, disk I/O, processes. */
|
||||
async function counters(isDisk) {
|
||||
const [stat, net, io] = await Promise.all([
|
||||
readText("/proc/stat"),
|
||||
readText("/proc/net/dev"),
|
||||
readText("/proc/diskstats"),
|
||||
]);
|
||||
return {
|
||||
at: Date.now(),
|
||||
cpu: parseCpuStat(stat),
|
||||
net: parseNetDev(net),
|
||||
io: parseDiskstats(io, isDisk),
|
||||
procs: await processes(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Two samples `SAMPLE_MS` apart: per-user CPU and memory (`usage`), and from
|
||||
* the same window per-core CPU, network and disk I/O rates and the busiest
|
||||
* processes machine-wide.
|
||||
*/
|
||||
async function sampleWindow(mem) {
|
||||
const disksSet = wholeDisks();
|
||||
const isDisk = (n) => !disksSet || disksSet.has(n);
|
||||
const s0 = await counters(isDisk);
|
||||
await deps.wait(SAMPLE_MS);
|
||||
const t1 = await cpuTimes();
|
||||
const p1 = await processes();
|
||||
const mem = await memory();
|
||||
const s1 = await counters(isDisk);
|
||||
const ms = s1.at - s0.at;
|
||||
const cores = s1.cpu.cores.map((c, i) => (s0.cpu.cores[i] ? busyPct(s0.cpu.cores[i], c) : 0));
|
||||
const use = await usage(s0.cpu.all, s1.cpu.all, s0.procs, s1.procs, mem);
|
||||
return {
|
||||
usage: use ? use.usage : null,
|
||||
topProcesses: use ? use.top : [],
|
||||
cores,
|
||||
network: rates(s0.net, s1.net, ms, ["rx", "tx"]),
|
||||
diskIo: rates(s0.io, s1.io, ms, ["read", "write"]),
|
||||
};
|
||||
}
|
||||
|
||||
/** Per-user CPU and memory from two samples (`usage`) and the busiest processes (`top`). */
|
||||
async function usage(t0, t1, p0, p1, mem) {
|
||||
if (!t0 || !t1) return null;
|
||||
const totalDelta = Math.max(1, t1.total - t0.total);
|
||||
const hostUid = typeof process.getuid === "function" ? process.getuid() : -1;
|
||||
@@ -286,6 +477,7 @@ async function usage() {
|
||||
const byId = new Map(users.map((u) => [u.id, u]));
|
||||
const ownerCache = new Map();
|
||||
const groups = new Map();
|
||||
const all = [];
|
||||
const group = (key, extra) => {
|
||||
if (!groups.has(key))
|
||||
groups.set(key, { key, cpu: 0, memory: 0, processes: 0, top: [], ...extra });
|
||||
@@ -301,7 +493,7 @@ async function usage() {
|
||||
const member = deps.memberNameForUid(p.uid);
|
||||
if (member && byUsername.has(member)) {
|
||||
const u = byUsername.get(member);
|
||||
g = group(`user:${u.id}`, { kind: "user", userId: u.id, username: u.username });
|
||||
g = group(`user:${u.id}`, userExtra(u));
|
||||
} else if (p.uid === hostUid) {
|
||||
const laneId = await laneIdOf(pid);
|
||||
let ownerId = null;
|
||||
@@ -318,9 +510,7 @@ async function usage() {
|
||||
ownerId = ownerCache.get(laneId);
|
||||
}
|
||||
const u = ownerId != null ? byId.get(ownerId) : null;
|
||||
g = u
|
||||
? group(`user:${u.id}`, { kind: "user", userId: u.id, username: u.username })
|
||||
: group("host", { kind: "host" });
|
||||
g = u ? group(`user:${u.id}`, userExtra(u)) : group("host", { kind: "host" });
|
||||
} else {
|
||||
g = group("system", { kind: "system" });
|
||||
}
|
||||
@@ -329,6 +519,7 @@ async function usage() {
|
||||
g.memory += p.rss;
|
||||
g.processes += 1;
|
||||
g.top.push({ pid, name: p.comm, cpu, memory: p.rss });
|
||||
all.push({ pid, name: p.comm, cpu, memory: p.rss, group: g.key });
|
||||
})
|
||||
);
|
||||
|
||||
@@ -342,9 +533,7 @@ async function usage() {
|
||||
if (!groups.has(`user:${u.id}`))
|
||||
list.push({
|
||||
key: `user:${u.id}`,
|
||||
kind: "user",
|
||||
userId: u.id,
|
||||
username: u.username,
|
||||
...userExtra(u),
|
||||
cpu: 0,
|
||||
memory: 0,
|
||||
memoryPercent: 0,
|
||||
@@ -353,24 +542,40 @@ async function usage() {
|
||||
});
|
||||
}
|
||||
list.sort((a, b) => b.cpu - a.cpu || b.memory - a.memory);
|
||||
all.sort((a, b) => b.cpu - a.cpu || b.memory - a.memory);
|
||||
return {
|
||||
cpu: round(((totalDelta - (t1.idle - t0.idle)) / totalDelta) * 100),
|
||||
sampleMs: SAMPLE_MS,
|
||||
groups: list,
|
||||
usage: {
|
||||
cpu: round(((totalDelta - (t1.idle - t0.idle)) / totalDelta) * 100),
|
||||
sampleMs: SAMPLE_MS,
|
||||
groups: list,
|
||||
},
|
||||
top: all.slice(0, TOP_PROCESSES).map((t) => ({ ...t, cpu: round(t.cpu) })),
|
||||
};
|
||||
}
|
||||
|
||||
const round = (n) => Math.round(n * 10) / 10;
|
||||
|
||||
/** A user's group fields: id, username and display name (every viewer reads them). */
|
||||
const userExtra = (u) => ({
|
||||
kind: "user",
|
||||
userId: u.id,
|
||||
username: u.username,
|
||||
displayName: u.displayName ?? null,
|
||||
});
|
||||
|
||||
async function collect() {
|
||||
const linux = process.platform === "linux";
|
||||
const cpus = os.cpus();
|
||||
const [mem, diskList, gpuList, use] = await Promise.all([
|
||||
memory(),
|
||||
const mem = await memory();
|
||||
const [diskList, gpuList, win, temps, cpuinfo, psi] = await Promise.all([
|
||||
linux ? disks() : [],
|
||||
linux ? gpus() : [],
|
||||
linux ? usage() : null,
|
||||
linux ? sampleWindow(mem) : null,
|
||||
linux ? temperatures() : [],
|
||||
linux ? readText("/proc/cpuinfo") : null,
|
||||
linux ? pressure() : null,
|
||||
]);
|
||||
const pmem = process.memoryUsage();
|
||||
return {
|
||||
at: new Date().toISOString(),
|
||||
hostname: os.hostname(),
|
||||
@@ -381,12 +586,31 @@ async function collect() {
|
||||
arch: os.arch(),
|
||||
},
|
||||
uptime: os.uptime(),
|
||||
cpu: { model: cpus[0]?.model?.trim() || null, cores: cpus.length, load: os.loadavg() },
|
||||
bootTime: new Date(Date.now() - os.uptime() * 1000).toISOString(),
|
||||
cpu: {
|
||||
model: cpus[0]?.model?.trim() || null,
|
||||
cores: cpus.length,
|
||||
load: os.loadavg(),
|
||||
physicalCores: physicalCores(cpuinfo),
|
||||
perCore: win ? win.cores : [],
|
||||
},
|
||||
memory: mem,
|
||||
disks: diskList,
|
||||
homesMount: mountOf(diskList, deps.homesRoot()),
|
||||
gpus: gpuList,
|
||||
usage: use,
|
||||
usage: win ? win.usage : null,
|
||||
network: win ? win.network : [],
|
||||
diskIo: win ? win.diskIo : [],
|
||||
temperatures: temps,
|
||||
pressure: psi,
|
||||
topProcesses: win ? win.topProcesses : [],
|
||||
process: {
|
||||
pid: process.pid,
|
||||
node: process.version,
|
||||
uptime: Math.round(process.uptime()),
|
||||
rss: pmem.rss,
|
||||
heapUsed: pmem.heapUsed,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -409,9 +633,134 @@ async function hostStats() {
|
||||
return inflight;
|
||||
}
|
||||
|
||||
// ── History ────────────────────────────────────────────────────────────────
|
||||
// An in-memory ring of machine-level samples, one every HISTORY_EVERY_MS for
|
||||
// HISTORY_KEEP_MS, started by every server on Linux (server/index.js). It
|
||||
// starts empty at every server start: nothing is written to disk.
|
||||
|
||||
let history = [];
|
||||
let historyTimer = null;
|
||||
|
||||
/** One history point from a snapshot. */
|
||||
function historyPoint(s) {
|
||||
const users = {};
|
||||
for (const g of s.usage?.groups || []) {
|
||||
if (g.kind === "user" && (g.cpu || g.memory))
|
||||
users[g.userId] = { cpu: g.cpu, memory: g.memory };
|
||||
}
|
||||
return {
|
||||
at: s.at,
|
||||
cpu: s.usage ? s.usage.cpu : null,
|
||||
memory: round((s.memory.used / Math.max(1, s.memory.total)) * 100),
|
||||
memoryUsed: s.memory.used,
|
||||
swapUsed: s.memory.swapUsed,
|
||||
load1: round(s.cpu.load[0] || 0),
|
||||
users,
|
||||
};
|
||||
}
|
||||
|
||||
/** Take one sample now (reusing a fresh snapshot) and drop what is too old. */
|
||||
async function sampleHistory() {
|
||||
try {
|
||||
const point = historyPoint(await hostStats());
|
||||
const cutoff = Date.now() - HISTORY_KEEP_MS;
|
||||
history = [...history.filter((p) => Date.parse(p.at) >= cutoff), point];
|
||||
} catch {
|
||||
/* a failed sample is a gap, never a crash */
|
||||
}
|
||||
}
|
||||
|
||||
/** Start the sampler (Linux only; idempotent). The timer never holds the process open. */
|
||||
function startHistory() {
|
||||
if (historyTimer || process.platform !== "linux") return;
|
||||
void sampleHistory();
|
||||
historyTimer = setInterval(() => void sampleHistory(), HISTORY_EVERY_MS);
|
||||
historyTimer.unref?.();
|
||||
}
|
||||
|
||||
function stopHistory() {
|
||||
if (historyTimer) clearInterval(historyTimer);
|
||||
historyTimer = null;
|
||||
}
|
||||
|
||||
const RANGES = { "1h": 3600_000, "6h": 6 * 3600_000, "24h": 24 * 3600_000 };
|
||||
|
||||
/**
|
||||
* The points of the last `range` (`1h`, `6h`, `24h`; else 1h), averaged down
|
||||
* to at most HISTORY_MAX_POINTS. `startedAt` = the oldest point kept, so the
|
||||
* UI can say the history begins there.
|
||||
*/
|
||||
function hostHistory(range) {
|
||||
const span = RANGES[range] || RANGES["1h"];
|
||||
const since = Date.now() - span;
|
||||
const pts = history.filter((p) => Date.parse(p.at) >= since);
|
||||
return {
|
||||
range: RANGES[range] ? range : "1h",
|
||||
intervalMs: HISTORY_EVERY_MS,
|
||||
running: historyTimer != null,
|
||||
startedAt: history[0]?.at ?? null,
|
||||
points: downsample(pts, HISTORY_MAX_POINTS),
|
||||
};
|
||||
}
|
||||
|
||||
/** Average consecutive points into at most `max` buckets (numbers only; `at` = the bucket's last). */
|
||||
function downsample(pts, max) {
|
||||
if (pts.length <= max) return pts;
|
||||
const per = Math.ceil(pts.length / max);
|
||||
const out = [];
|
||||
for (let i = 0; i < pts.length; i += per) {
|
||||
const b = pts.slice(i, i + per);
|
||||
const avg = (k) => {
|
||||
const v = b.map((p) => p[k]).filter((x) => x != null);
|
||||
return v.length ? round(v.reduce((a, x) => a + x, 0) / v.length) : null;
|
||||
};
|
||||
const users = {};
|
||||
for (const p of b)
|
||||
for (const [id, u] of Object.entries(p.users)) {
|
||||
const t = (users[id] ||= { cpu: 0, memory: 0 });
|
||||
t.cpu += u.cpu / b.length;
|
||||
t.memory += u.memory / b.length;
|
||||
}
|
||||
for (const u of Object.values(users)) {
|
||||
u.cpu = round(u.cpu);
|
||||
u.memory = Math.round(u.memory);
|
||||
}
|
||||
out.push({
|
||||
at: b[b.length - 1].at,
|
||||
cpu: avg("cpu"),
|
||||
memory: avg("memory"),
|
||||
memoryUsed: Math.round(avg("memoryUsed") ?? 0),
|
||||
swapUsed: Math.round(avg("swapUsed") ?? 0),
|
||||
load1: avg("load1"),
|
||||
users,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function __setDeps(over) {
|
||||
deps = { ...deps, ...over };
|
||||
cache = null;
|
||||
}
|
||||
|
||||
module.exports = { hostStats, mountOf, __setDeps };
|
||||
/** Tests: replace the history ring. */
|
||||
function __setHistory(points) {
|
||||
history = points;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
hostStats,
|
||||
hostHistory,
|
||||
startHistory,
|
||||
stopHistory,
|
||||
mountOf,
|
||||
parseCpuStat,
|
||||
parsePressure,
|
||||
parseNetDev,
|
||||
parseDiskstats,
|
||||
physicalCores,
|
||||
temperatures,
|
||||
downsample,
|
||||
__setDeps,
|
||||
__setHistory,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* @file loop-delay.js
|
||||
* @description How long this server's event loop was held up — what an
|
||||
* operator sees as a laggy dashboard or terminal: every request, hook and
|
||||
* keystroke echo waits for it. One `monitorEventLoopDelay` histogram for the
|
||||
* whole process, sampled every 20 ms and reset each minute, so `loopDelay()`
|
||||
* describes the last full minute (the running one until the first minute
|
||||
* ends). Read by `GET /api/metrics` (Prometheus gauges) and
|
||||
* `GET /api/settings/info` (the System Health score).
|
||||
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
|
||||
*/
|
||||
|
||||
const { monitorEventLoopDelay } = require("perf_hooks");
|
||||
|
||||
const WINDOW_MS = 60_000;
|
||||
const histogram = monitorEventLoopDelay({ resolution: 20 });
|
||||
histogram.enable();
|
||||
|
||||
const read = () => ({
|
||||
p50: histogram.percentile(50),
|
||||
p99: histogram.percentile(99),
|
||||
max: histogram.max,
|
||||
});
|
||||
|
||||
let lastWindow = null; // the last full minute, reported until the next one ends
|
||||
setInterval(() => {
|
||||
lastWindow = read();
|
||||
histogram.reset();
|
||||
}, WINDOW_MS).unref();
|
||||
|
||||
/** `{p50, p99, max}` in nanoseconds over the last minute. */
|
||||
function loopDelay() {
|
||||
return lastWindow || read();
|
||||
}
|
||||
|
||||
module.exports = { loopDelay, WINDOW_MS };
|
||||
@@ -8,7 +8,9 @@
|
||||
* A Claude session counts only for the hours one of THIS account's runs held
|
||||
* it (start hour to end hour), so a session resumed on another account is
|
||||
* not booked twice; Codex has no hourly rows, so its session counts whole
|
||||
* once a run of the account started in the window, as before.
|
||||
* once a run of the account started in the window, as before. With `ownerId`
|
||||
* only the runs of that user's workspaces count — one user's share of the
|
||||
* account, for the Administration page's user detail.
|
||||
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
|
||||
*/
|
||||
|
||||
@@ -19,21 +21,28 @@ const tokensOf = (r) =>
|
||||
|
||||
/**
|
||||
* `{spend5h, spend7d, tokens5h, tokens7d}` of `account` on `tool`. With
|
||||
* `wholeSession`, a session's every row counts in a window its run started in.
|
||||
* `wholeSession`, a session's every row counts in a window its run started in;
|
||||
* with `ownerId`, only runs on that user's workspaces.
|
||||
*/
|
||||
function accountUsage(tool, account, { wholeSession = false, now = Date.now() } = {}) {
|
||||
function accountUsage(
|
||||
tool,
|
||||
account,
|
||||
{ wholeSession = false, now = Date.now(), ownerId = null } = {}
|
||||
) {
|
||||
const { db, stmts } = require("../db");
|
||||
const { calculateCost, hourlyTokenRows } = require("../routes/pricing");
|
||||
const since7d = new Date(now - WINDOWS["7d"] * 3600e3).toISOString();
|
||||
const owned = ownerId == null ? "" : "AND lane_id IN (SELECT id FROM lanes WHERE owner_id = ?)";
|
||||
const ownedArgs = ownerId == null ? [] : [ownerId];
|
||||
const runs = db
|
||||
.prepare(
|
||||
`SELECT session_id, substr(started_at, 1, 13) AS start_hour, started_at,
|
||||
substr(COALESCE(ended_at, '9999'), 1, 13) AS end_hour
|
||||
FROM dashboard_runs
|
||||
WHERE tool = ? AND account = ? AND session_id IS NOT NULL
|
||||
AND COALESCE(ended_at, '9999') >= ?`
|
||||
AND COALESCE(ended_at, '9999') >= ? ${owned}`
|
||||
)
|
||||
.all(tool, account, since7d);
|
||||
.all(tool, account, since7d, ...ownedArgs);
|
||||
const out = { spend5h: 0, spend7d: 0, tokens5h: 0, tokens7d: 0 };
|
||||
if (runs.length === 0) return out;
|
||||
const runsOf = new Map();
|
||||
@@ -41,8 +50,8 @@ function accountUsage(tool, account, { wholeSession = false, now = Date.now() }
|
||||
const rows = hourlyTokenRows(
|
||||
`WHERE s.id IN (SELECT session_id FROM dashboard_runs
|
||||
WHERE tool = ? AND account = ? AND session_id IS NOT NULL
|
||||
AND COALESCE(ended_at, '9999') >= ?)`,
|
||||
[tool, account, since7d]
|
||||
AND COALESCE(ended_at, '9999') >= ? ${owned})`,
|
||||
[tool, account, since7d, ...ownedArgs]
|
||||
);
|
||||
const rules = stmts.listPricing.all();
|
||||
for (const [label, hours] of Object.entries(WINDOWS)) {
|
||||
|
||||
Reference in New Issue
Block a user