/** * @file host-stats.js * @description The Dashboard System Health tab's machine cards (`GET /api/host`, * every signed-in user): the machine's hardware and load, and how much of its CPU and memory * each AgentHub user is using right now. Read-only — nothing here signals, * renices or kills a process. * * Hardware comes from `os` plus `/proc` (meminfo, stat, mounts) and `statfs` * per real mounted file system (HDD/SSD from `/sys/block//queue/rotational`); * a GPU only when `nvidia-smi` answers within 3 s. Usage is two `/proc` * samples `SAMPLE_MS` apart: a process's CPU % is its utime+stime delta over * the whole machine's jiffies delta (100 % = every core busy), its memory its * RSS. A process is attributed, in order, to: the AgentHub user of its uid * (`ah-` under S3, `memberNameForUid`); for a process of the host * account, the OWNER of the workspace named by its inherited * `AGENTHUB_LANE_ID` (every pane gets it through tmux `-e`, so an agent's * children carry it too); else `host` (the dashboard and everything else the * host account runs) or `system` (root and every other account). Only the * kernel's `comm` name is reported for a process — never its argv or env, * which can hold a prompt or a key. The same sample window also gives per-core * CPU % (`/proc/stat` cpuN), network rx/tx per interface (`/proc/net/dev`, no * `lo`), disk read/write per whole disk (`/proc/diskstats`), and the ten * busiest processes machine-wide with their group; temperatures come from * `/sys/class/hwmon`, physical cores from `/proc/cpuinfo`, and the dashboard's * own pid/Node/uptime/RSS from `process`, and pressure stall information * (`/proc/pressure/{cpu,memory,io}`, PSI — the share of time tasks waited on * that resource, i.e. how laggy the machine feels; `null` on a kernel without * it). Every read is async, and one sample is * shared by concurrent callers and cached for `FRESH_MS`. Linux only: other * platforms get the `os` facts with `usage: null` and empty lists. * * History (`GET /api/host/history`): `startHistory` (every server on Linux, * server/index.js) keeps an in-memory ring of machine CPU/memory/swap/load1 * and per-user CPU/memory, one point every 30 s for 24 h, from the cached * snapshot; it starts empty at every server start and is never written to disk. * @author Nguyễn Ngọc Trí Vĩ */ const fs = require("node:fs"); const fsp = require("node:fs/promises"); const os = require("node:os"); const path = require("node:path"); const { execFile } = require("node:child_process"); const SAMPLE_MS = 1000; const FRESH_MS = 3000; const TOP_PER_GROUP = 5; const TOP_PROCESSES = 10; const HISTORY_EVERY_MS = 30_000; const HISTORY_KEEP_MS = 24 * 3600_000; const HISTORY_MAX_POINTS = 360; // File systems that hold data; pseudo and loop mounts are left out. const REAL_FS = new Set([ "ext2", "ext3", "ext4", "xfs", "btrfs", "zfs", "f2fs", "vfat", "exfat", "ntfs", "ntfs3", "fuseblk", "nfs", "nfs4", "cifs", ]); const readText = (file) => fsp.readFile(file, "utf8").catch(() => null); /** `/proc/meminfo` in bytes: total, available, swap. */ async function memory() { const text = await readText("/proc/meminfo"); const kb = (key) => { const m = text && new RegExp(`^${key}:\\s+(\\d+)`, "m").exec(text); return m ? Number(m[1]) * 1024 : null; }; const total = kb("MemTotal") ?? os.totalmem(); const available = kb("MemAvailable") ?? os.freemem(); return { total, available, used: total - available, swapTotal: kb("SwapTotal") ?? 0, swapUsed: (kb("SwapTotal") ?? 0) - (kb("SwapFree") ?? 0), }; } /** * `/proc/stat`'s jiffies: the machine's `cpu` line and each `cpuN` line, each * `{ total, idle }` (idle = idle + iowait). `{ all: null, cores: [] }` when * there is no `cpu` line. */ function parseCpuStat(text) { const times = (line) => { const n = line.trim().split(/\s+/).slice(1).map(Number); return { total: n.reduce((a, b) => a + b, 0), idle: n[3] + (n[4] || 0) }; }; const lines = (text || "").split("\n"); const all = lines.find((l) => l.startsWith("cpu ")); return { all: all ? times(all) : null, cores: lines.filter((l) => /^cpu\d+ /.test(l)).map(times), }; } /** * One `/proc/pressure/` file: `{some10, some60, full10}` — the % of * the last 10 s / 60 s that at least one task (`some`) or every task (`full`) * was stalled on it. `null` when the file is missing or unreadable. */ function parsePressure(text) { if (!text) return null; const line = (kind) => { const m = new RegExp(`^${kind} avg10=([\\d.]+) avg60=([\\d.]+)`, "m").exec(text); return m ? { avg10: Number(m[1]), avg60: Number(m[2]) } : null; }; const some = line("some"); if (!some) return null; return { some10: some.avg10, some60: some.avg60, full10: line("full")?.avg10 ?? 0 }; } /** PSI for cpu, memory and io; `null` when the kernel has none of them. */ async function pressure() { const [cpu, memory, io] = await Promise.all( ["cpu", "memory", "io"].map((r) => readText(`/proc/pressure/${r}`).then(parsePressure)) ); return cpu || memory || io ? { cpu, memory, io } : null; } /** Busy % between two `{ total, idle }` readings. */ const busyPct = (a, b) => { const total = b.total - a.total; return total > 0 ? round(((total - (b.idle - a.idle)) / total) * 100) : 0; }; /** `/proc/net/dev`: `Map` in bytes, `lo` left out. */ function parseNetDev(text) { const out = new Map(); for (const line of (text || "").split("\n").slice(2)) { const i = line.indexOf(":"); if (i < 0) continue; const name = line.slice(0, i).trim(); if (!name || name === "lo") continue; const f = line .slice(i + 1) .trim() .split(/\s+/) .map(Number); out.set(name, { rx: f[0] || 0, tx: f[8] || 0 }); } return out; } /** * `/proc/diskstats`: `Map` in bytes (sectors × 512) * for whole disks only (`isDisk(name)`), never loop/ram/zram/optical. */ function parseDiskstats(text, isDisk = () => true) { const out = new Map(); for (const line of (text || "").split("\n")) { const f = line.trim().split(/\s+/); if (f.length < 10) continue; const name = f[2]; if (/^(loop|ram|zram|sr|fd)\d*/.test(name) || !isDisk(name)) continue; out.set(name, { read: Number(f[5]) * 512, write: Number(f[9]) * 512 }); } return out; } /** Whole disks: the names under `/sys/block` (partitions are not there). */ function wholeDisks() { try { return new Set(fs.readdirSync("/sys/block")); } catch { return null; } } /** Per-key `{a, b}` byte counters → per-second rates over `ms`. */ function rates(m0, m1, ms, keys) { const sec = Math.max(0.001, ms / 1000); const out = []; for (const [name, v1] of m1) { const v0 = m0.get(name); if (!v0) continue; const row = { name }; for (const k of keys) row[k] = Math.max(0, Math.round((v1[k] - v0[k]) / sec)); out.push(row); } return out; } /** * Temperatures from `/sys/class/hwmon/*`: `[{ chip, label, celsius }]`, one * per `tempN_input` (label from `tempN_label`, else `tempN`). [] when none. */ async function temperatures(root = "/sys/class/hwmon") { let chips; try { chips = await fsp.readdir(root); } catch { return []; } const out = []; for (const c of chips.sort()) { const dir = path.join(root, c); const chip = ((await readText(path.join(dir, "name"))) || c).trim(); let files; try { files = await fsp.readdir(dir); } catch { continue; } for (const f of files.filter((x) => /^temp\d+_input$/.test(x)).sort()) { const raw = Number(((await readText(path.join(dir, f))) || "").trim()); if (!Number.isFinite(raw) || raw === 0) continue; const id = f.replace("_input", ""); const label = ((await readText(path.join(dir, `${id}_label`))) || id).trim(); out.push({ chip, label, celsius: round(raw / 1000) }); } } return out; } /** Physical cores from `/proc/cpuinfo` (distinct physical id + core id), or null. */ function physicalCores(text) { const seen = new Set(); let phys = "0"; for (const line of (text || "").split("\n")) { const m = /^(physical id|core id)\s*:\s*(\d+)/.exec(line); if (!m) continue; if (m[1] === "physical id") phys = m[2]; else seen.add(`${phys}:${m[2]}`); } return seen.size || null; } /** `HDD`/`SSD` for a `/dev/...` device, or null when /sys cannot tell. */ function diskKind(device) { if (!device.startsWith("/dev/")) return null; let name = path.basename(device); try { name = path.basename(fs.realpathSync(device)); } catch { /* a mapper name that does not resolve: try it as is */ } // A partition's queue lives on its parent: sda1 → sda, nvme0n1p2 → nvme0n1. const candidates = [name, name.replace(/p?\d+$/, "")]; for (const c of candidates) { try { const v = fs.readFileSync(`/sys/block/${c}/queue/rotational`, "utf8").trim(); return v === "1" ? "HDD" : "SSD"; } catch { /* next */ } } return null; } /** Every real mounted file system, one row per device. */ async function disks() { const text = await readText("/proc/mounts"); if (!text) return []; const seen = new Set(); const out = []; for (const line of text.split("\n")) { const [device, rawMount, type] = line.split(" "); if (!device || !REAL_FS.has(type) || seen.has(device)) continue; if (device.startsWith("/dev/loop")) continue; const mount = rawMount.replace(/\\040/g, " "); if (mount.startsWith("/snap/") || mount.startsWith("/boot/efi")) continue; seen.add(device); try { const st = await fsp.statfs(mount); const total = st.blocks * st.bsize; if (!total) continue; const free = st.bavail * st.bsize; out.push({ device, mount, type, kind: diskKind(device), total, used: total - st.bfree * st.bsize, free, }); } catch { /* unreadable mount: skip */ } } return out; } /** The mount (from `list`) holding `p`: the longest mount point that prefixes it. */ function mountOf(list, p) { if (!p) return null; let best = null; for (const d of list) { const m = d.mount === "/" ? "/" : `${d.mount}/`; if ((p === d.mount || p.startsWith(m)) && (!best || d.mount.length > best.mount.length)) best = d; } return best ? best.mount : null; } /** `nvidia-smi` GPUs, or [] when there is none (or it does not answer in 3 s). */ function gpus() { return new Promise((resolve) => { execFile( "nvidia-smi", [ "--query-gpu=name,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits", ], { timeout: 3000, windowsHide: true }, (err, stdout) => { if (err) return resolve([]); resolve( String(stdout) .trim() .split("\n") .filter(Boolean) .map((l) => { const [name, util, used, total] = l.split(",").map((s) => s.trim()); return { name, utilization: Number(util) || 0, memoryUsed: (Number(used) || 0) * 1024 * 1024, memoryTotal: (Number(total) || 0) * 1024 * 1024, }; }) ); } ); }); } function distro() { try { const m = /^PRETTY_NAME="?([^"\n]*)"?/m.exec(fs.readFileSync("/etc/os-release", "utf8")); return m ? m[1] : null; } catch { return null; } } const PAGE = 4096; /** * One pass over `/proc`: `Map`. `stat`'s comm * is in parentheses and may itself hold spaces or `)`, so the fields are read * after the LAST `)`. */ async function processes() { const out = new Map(); let names; try { names = await fsp.readdir("/proc"); } catch { return out; } await Promise.all( names .filter((n) => /^\d+$/.test(n)) .map(async (pid) => { try { const [stat, st] = await Promise.all([ fsp.readFile(`/proc/${pid}/stat`, "utf8"), fsp.stat(`/proc/${pid}`), ]); const close = stat.lastIndexOf(")"); const comm = stat.slice(stat.indexOf("(") + 1, close); const f = stat.slice(close + 2).split(" "); // After comm: state(0) ppid(1) … utime(11) stime(12) … rss(21). out.set(Number(pid), { uid: st.uid, comm, ticks: Number(f[11]) + Number(f[12]), rss: Number(f[21]) * PAGE, }); } catch { /* exited meanwhile */ } }) ); return out; } /** The `AGENTHUB_LANE_ID` a host process inherited, or null. */ async function laneIdOf(pid) { try { const env = await fsp.readFile(`/proc/${pid}/environ`, "utf8"); const m = /(?:^|\0)AGENTHUB_LANE_ID=(\d+)(?:\0|$)/.exec(env); return m ? Number(m[1]) : null; } catch { return null; } } /** Defaults wired to the real modules; a test swaps them (`__setDeps`). */ let deps = { memberNameForUid: (uid) => require("./run-as").memberNameForUid(uid), laneOwner: (laneId) => { const { db } = require("../db"); const row = db.prepare("SELECT owner_id FROM lanes WHERE id = ?").get(laneId); return row ? row.owner_id : null; }, users: () => { try { return require("./auth").listUsers(); } catch { return []; } }, homesRoot: () => { try { const linux = require("./linux-accounts"); if (linux.s3Enabled()) return linux.serverConf().usersRoot; } catch { /* no S3 */ } return process.env.AGENTHUB_USERS_ROOT || os.homedir(); }, wait: (ms) => new Promise((r) => setTimeout(r, ms)), }; /** The counters one sample reads: CPU jiffies, network, disk I/O, processes. */ async function counters(isDisk) { const [stat, net, io] = await Promise.all([ readText("/proc/stat"), readText("/proc/net/dev"), readText("/proc/diskstats"), ]); return { at: Date.now(), cpu: parseCpuStat(stat), net: parseNetDev(net), io: parseDiskstats(io, isDisk), procs: await processes(), }; } /** * Two samples `SAMPLE_MS` apart: per-user CPU and memory (`usage`), and from * the same window per-core CPU, network and disk I/O rates and the busiest * processes machine-wide. */ async function sampleWindow(mem) { const disksSet = wholeDisks(); const isDisk = (n) => !disksSet || disksSet.has(n); const s0 = await counters(isDisk); await deps.wait(SAMPLE_MS); const s1 = await counters(isDisk); const ms = s1.at - s0.at; const cores = s1.cpu.cores.map((c, i) => (s0.cpu.cores[i] ? busyPct(s0.cpu.cores[i], c) : 0)); const use = await usage(s0.cpu.all, s1.cpu.all, s0.procs, s1.procs, mem); return { usage: use ? use.usage : null, topProcesses: use ? use.top : [], cores, network: rates(s0.net, s1.net, ms, ["rx", "tx"]), diskIo: rates(s0.io, s1.io, ms, ["read", "write"]), }; } /** Per-user CPU and memory from two samples (`usage`) and the busiest processes (`top`). */ async function usage(t0, t1, p0, p1, mem) { if (!t0 || !t1) return null; const totalDelta = Math.max(1, t1.total - t0.total); const hostUid = typeof process.getuid === "function" ? process.getuid() : -1; const users = deps.users(); const byUsername = new Map(users.map((u) => [u.username, u])); const byId = new Map(users.map((u) => [u.id, u])); const ownerCache = new Map(); const groups = new Map(); const all = []; const group = (key, extra) => { if (!groups.has(key)) groups.set(key, { key, cpu: 0, memory: 0, processes: 0, top: [], ...extra }); return groups.get(key); }; await Promise.all( [...p1].map(async ([pid, p]) => { const before = p0.get(pid); // A process born between the samples: all its ticks are in the window. const ticks = Math.max(0, p.ticks - (before ? before.ticks : 0)); let g = null; const member = deps.memberNameForUid(p.uid); if (member && byUsername.has(member)) { const u = byUsername.get(member); g = group(`user:${u.id}`, userExtra(u)); } else if (p.uid === hostUid) { const laneId = await laneIdOf(pid); let ownerId = null; if (laneId != null) { if (!ownerCache.has(laneId)) { let o = null; try { o = deps.laneOwner(laneId); } catch { o = null; } ownerCache.set(laneId, o); } ownerId = ownerCache.get(laneId); } const u = ownerId != null ? byId.get(ownerId) : null; g = u ? group(`user:${u.id}`, userExtra(u)) : group("host", { kind: "host" }); } else { g = group("system", { kind: "system" }); } const cpu = (ticks / totalDelta) * 100; g.cpu += cpu; g.memory += p.rss; g.processes += 1; g.top.push({ pid, name: p.comm, cpu, memory: p.rss }); all.push({ pid, name: p.comm, cpu, memory: p.rss, group: g.key }); }) ); const list = [...groups.values()].map((g) => { g.top.sort((a, b) => b.cpu - a.cpu || b.memory - a.memory); g.top = g.top.slice(0, TOP_PER_GROUP).map((t) => ({ ...t, cpu: round(t.cpu) })); return { ...g, cpu: round(g.cpu), memoryPercent: round((g.memory / mem.total) * 100) }; }); // Every user appears, idle ones with zeros, so the table is the whole roster. for (const u of users) { if (!groups.has(`user:${u.id}`)) list.push({ key: `user:${u.id}`, ...userExtra(u), cpu: 0, memory: 0, memoryPercent: 0, processes: 0, top: [], }); } list.sort((a, b) => b.cpu - a.cpu || b.memory - a.memory); all.sort((a, b) => b.cpu - a.cpu || b.memory - a.memory); return { usage: { cpu: round(((totalDelta - (t1.idle - t0.idle)) / totalDelta) * 100), sampleMs: SAMPLE_MS, groups: list, }, top: all.slice(0, TOP_PROCESSES).map((t) => ({ ...t, cpu: round(t.cpu) })), }; } const round = (n) => Math.round(n * 10) / 10; /** A user's group fields: id, username and display name (every viewer reads them). */ const userExtra = (u) => ({ kind: "user", userId: u.id, username: u.username, displayName: u.displayName ?? null, }); async function collect() { const linux = process.platform === "linux"; const cpus = os.cpus(); const mem = await memory(); const [diskList, gpuList, win, temps, cpuinfo, psi] = await Promise.all([ linux ? disks() : [], linux ? gpus() : [], linux ? sampleWindow(mem) : null, linux ? temperatures() : [], linux ? readText("/proc/cpuinfo") : null, linux ? pressure() : null, ]); const pmem = process.memoryUsage(); return { at: new Date().toISOString(), hostname: os.hostname(), os: { platform: process.platform, release: os.release(), distro: linux ? distro() : null, arch: os.arch(), }, uptime: os.uptime(), bootTime: new Date(Date.now() - os.uptime() * 1000).toISOString(), cpu: { model: cpus[0]?.model?.trim() || null, cores: cpus.length, load: os.loadavg(), physicalCores: physicalCores(cpuinfo), perCore: win ? win.cores : [], }, memory: mem, disks: diskList, homesMount: mountOf(diskList, deps.homesRoot()), gpus: gpuList, usage: win ? win.usage : null, network: win ? win.network : [], diskIo: win ? win.diskIo : [], temperatures: temps, pressure: psi, topProcesses: win ? win.topProcesses : [], process: { pid: process.pid, node: process.version, uptime: Math.round(process.uptime()), rss: pmem.rss, heapUsed: pmem.heapUsed, }, }; } let cache = null; // { at, value } let inflight = null; /** The current snapshot: cached for `FRESH_MS`, one sample shared by every caller. */ async function hostStats() { if (cache && Date.now() - cache.at < FRESH_MS) return cache.value; if (!inflight) { inflight = collect() .then((value) => { cache = { at: Date.now(), value }; return value; }) .finally(() => { inflight = null; }); } return inflight; } // ── History ──────────────────────────────────────────────────────────────── // An in-memory ring of machine-level samples, one every HISTORY_EVERY_MS for // HISTORY_KEEP_MS, started by every server on Linux (server/index.js). It // starts empty at every server start: nothing is written to disk. let history = []; let historyTimer = null; /** One history point from a snapshot. */ function historyPoint(s) { const users = {}; for (const g of s.usage?.groups || []) { if (g.kind === "user" && (g.cpu || g.memory)) users[g.userId] = { cpu: g.cpu, memory: g.memory }; } return { at: s.at, cpu: s.usage ? s.usage.cpu : null, memory: round((s.memory.used / Math.max(1, s.memory.total)) * 100), memoryUsed: s.memory.used, swapUsed: s.memory.swapUsed, load1: round(s.cpu.load[0] || 0), users, }; } /** Take one sample now (reusing a fresh snapshot) and drop what is too old. */ async function sampleHistory() { try { const point = historyPoint(await hostStats()); const cutoff = Date.now() - HISTORY_KEEP_MS; history = [...history.filter((p) => Date.parse(p.at) >= cutoff), point]; } catch { /* a failed sample is a gap, never a crash */ } } /** Start the sampler (Linux only; idempotent). The timer never holds the process open. */ function startHistory() { if (historyTimer || process.platform !== "linux") return; void sampleHistory(); historyTimer = setInterval(() => void sampleHistory(), HISTORY_EVERY_MS); historyTimer.unref?.(); } function stopHistory() { if (historyTimer) clearInterval(historyTimer); historyTimer = null; } const RANGES = { "1h": 3600_000, "6h": 6 * 3600_000, "24h": 24 * 3600_000 }; /** * The points of the last `range` (`1h`, `6h`, `24h`; else 1h), averaged down * to at most HISTORY_MAX_POINTS. `startedAt` = the oldest point kept, so the * UI can say the history begins there. */ function hostHistory(range) { const span = RANGES[range] || RANGES["1h"]; const since = Date.now() - span; const pts = history.filter((p) => Date.parse(p.at) >= since); return { range: RANGES[range] ? range : "1h", intervalMs: HISTORY_EVERY_MS, running: historyTimer != null, startedAt: history[0]?.at ?? null, points: downsample(pts, HISTORY_MAX_POINTS), }; } /** Average consecutive points into at most `max` buckets (numbers only; `at` = the bucket's last). */ function downsample(pts, max) { if (pts.length <= max) return pts; const per = Math.ceil(pts.length / max); const out = []; for (let i = 0; i < pts.length; i += per) { const b = pts.slice(i, i + per); const avg = (k) => { const v = b.map((p) => p[k]).filter((x) => x != null); return v.length ? round(v.reduce((a, x) => a + x, 0) / v.length) : null; }; const users = {}; for (const p of b) for (const [id, u] of Object.entries(p.users)) { const t = (users[id] ||= { cpu: 0, memory: 0 }); t.cpu += u.cpu / b.length; t.memory += u.memory / b.length; } for (const u of Object.values(users)) { u.cpu = round(u.cpu); u.memory = Math.round(u.memory); } out.push({ at: b[b.length - 1].at, cpu: avg("cpu"), memory: avg("memory"), memoryUsed: Math.round(avg("memoryUsed") ?? 0), swapUsed: Math.round(avg("swapUsed") ?? 0), load1: avg("load1"), users, }); } return out; } function __setDeps(over) { deps = { ...deps, ...over }; cache = null; } /** Tests: replace the history ring. */ function __setHistory(points) { history = points; } module.exports = { hostStats, hostHistory, startHistory, stopHistory, mountOf, parseCpuStat, parsePressure, parseNetDev, parseDiskstats, physicalCores, temperatures, downsample, __setDeps, __setHistory, };